{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9bcfbc65",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yuhangya/Library/Python/3.8/lib/python/site-packages/cvxpy/problems/problem.py:1278: UserWarning: Solution may be inaccurate. Try another solver, adjusting the solver settings, or solve with verbose=True for more information.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x16d4845e0>]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAg5UlEQVR4nO3deXxV5Z3H8c+PJOxhD1tICPu+h01ti4wLIkXGqsVdtMO0o7XtqLi3denUpWpRVIoV0S7aKloQF2QTXEF2CJANgiGEJBAIgey5z/yR60xKIwS4ycm99/t+ve4r9yzJ+T3h8uXwnOecx5xziIhI8GvkdQEiIhIYCnQRkRChQBcRCREKdBGREKFAFxEJEZFeHbhDhw4uISHBq8OLiASlDRs2HHTOxdS0zbNAT0hIYP369V4dXkQkKJnZ3m/bpi4XEZEQoUAXEQkRCnQRkRChQBcRCREKdBGREKFAFxEJEQp0EZEQoUAXEaknBcXlzFuTzpe7D9XJz/fsxiIRkXDh8zne2rCPxz/cxaHjZfz4e70Y17N9wI+jQBcRqUNb9x3hl4uS2Jx5hFHd2/LKjNEM7damTo6lQBcRqQO5R0t4ZnkKb3yVSfsWTXjqymFcPjIWM6uzYyrQRUQC6NCxUl78OJ0/r91LRaVjxjk9+PmFfWjVNKrOj61AFxEJgIpKH3NXpzN39W6KyiqYNiKW2yf2IaFDi3qrQYEuInKW1qTk8dgHu9iRfZSLBnZi1qT+9O7Yst7rUKCLiJyhTV8f5rEPdrF2Tz7x7Zrz3NUj+P6wrp7Vo0AXETlNuUdLePzDZBZu3EfH6CY8cOkArh/fnSaREZ7WpUAXEamlY6UVzFudzkuf7KHS5/jJhF7cen5vWjZpGFHaMKoQEWngvtx9iJ++vom8wlKmDO3CXRf3o3v7+rvgWRsKdBGRkygoLmf28lRe/SKD7u2b89INiQyPa+N1WTVSoIuI1CCvsJRnlqfw7ub9HCurYProeO6b3J/oehhPfqYU6CIi1ZRWVLLgswyeW5lGSXklU4d15ebzejA4trXXpZ2SAl1EBCir8LFocxbPr0oj41ARE/t35P5LB9Arpv7Hk58pBbqIhLVKn2Phxn3MW7ObtNxj9O8czas3j+F7fWO8Lu20KdBFJGxt2JvPrxYnsT3rKH06tmTe9aO4cGCnOn2AVl1SoItI2Mk9WsJvP9jFO5uy6NyqKc9ePYLvD+0StEH+DQW6iISNsgofr3y2h2dXpFJe6bj1/F7814TetGggNwadrdBohYjISRSWlPP39ftY8PkeMvOLuWBARx6cMrDB3Rh0thToIhKyKn2Ov32Vye+Xp5BbWMrohLY8ctlgJvTr6HVpdeKUgW5mccBrQCfAAfOcc7NP2GcCsAjY41/1tnPu4YBWKiJSS6UVlby3NZtXPstgW1YBw+PaMPf6UYyMb+t1aXWqNmfoFcAdzrmNZhYNbDCzZc65HSfs94lzbkrgSxQRqb2Pk3N56N0d7Dl4nLh2zULmgmdtnDLQnXPZQLb/faGZ7QRigRMDXUTEM9uzCnj8w118knqQnh1aMP+mRM7v1zEsgvwbp9WHbmYJwAhgbQ2bx5vZFmA/cKdzLqmG758JzASIj48/7WJFRE50pKiMJ5Ym8/q6r2nTLIoHLh3ADeMTaBzZyOvS6l2tA93MWgILgZ87546esHkj0N05d8zMJgP/APqc+DOcc/OAeQCJiYnuTIsWESmv9PG3rzJ5elkKBcXl3HJuD26/oH4mY26oahXoZhZFVZj/xTn39onbqwe8c+59M3vBzDo45w4GrlQREfD5HEu2ZfP0R8lkHCpidEJbHpo6mIFdW3ldmudqM8rFgJeBnc65p79ln85AjnPOmdkYoBFwKKCVikhYq6j08c6mLOauTic97zj9OkXz8o2JTOwfXv3kJ1ObM/RzgeuBbWa22b/uPiAewDk3F7gC+ImZVQDFwHTnnLpUROSsVVT6+Nv6TP74yR72HDzOwC6tmD19OFOGdiWikYK8utqMcvkUOOlvzTk3B5gTqKJERABWp+Txm/d2kJJzjOFxbXjh2pFcMrizzsi/he4UFZEGJzWnkEff28nqlDy6t2/O3OtGcfGg4H0KYn1RoItIg3HoWNW0b6+vy6R544iwHoJ4JhToIuK5kvJKFnyewfMr0ygqr+S6sfH87IK+tGvR2OvSgooCXUQ84/M5Fm/Zz5NLk8k6UszE/h25b/IAencMnmnfGhIFuoh44ov0Q/zP+zvZllXAwC6teOKKoZzbu4PXZQU1BbqI1Ku03EIe+2AXy3fm0qV1U56+ahjThsfSSEMQz5oCXUTqRV5hKb9fnsIbX2XSLCqCWZP6cfO5PWgaFeF1aSFDgS4idaq4rJKXP93Nix+nU1rh47qx8dz+b31o37KJ16WFHAW6iNSJ46UVvLMpizkr0zhwtISLB3Xi7kn96RmjC551RYEuIgHlnOOjHTn8enES2QUlDItrw3PXjGB0QjuvSwt5CnQRCZivMvJ5cmky6/bk079zNM/8cDhje7TTHZ71RIEuImdt674jPPVRCqtT8ujQsgm//v5Arh3XnagI3eFZnxToInLG0nILeXJpMkuTcmjTPIp7LunPjeMTaNZYI1e8oEAXkdNWVFbBS2v2MGdVKk0iI/j5BX245bweRIfxbEENgQJdRGqtpLySP3+5l7mr0zl4rIwpQ7vw0NRBGoLYQCjQReSUnHMsTcrh0fd2sO9wMef17sAvLuzDqO4audKQKNBF5KTScgt56N0dfJJ6kH6dovnrj8Zyjp650iAp0EWkRrmFJbywKp0/f7mX5o0j+PX3B3LduO5EauRKg6VAF5F/UlpRyfxPM3huZSqlFT6uSuzGnRf1Uz95EFCgiwgAlT7HWxsymbMqjcz8Yi4c2In7Jg+gR4cWXpcmtaRAFxHWZ+Tzq8VJJO0/yrBurXl02hC+1zfG67LkNCnQRcJY7tESfvvBLt7ZlEWX1k2Zc80ILh3SRbfqBykFukgY8vkcf1m7l8c/TKaswset5/fi1vN707yxIiGY6U9PJMx8faiIWQu38OXufL7TpwOPXDaYBPWThwQFukiYcM7x5oZ9/HpxEhFmPHb5EH44Ok7dKyFEgS4SBlJzCnn0vZ2sTsljXM92PHXVcGLbNPO6LAkwBbpICMsrLOWZ5Sm8se5rWjSJ5MEpA7npnAQiNCFzSDploJtZHPAa0AlwwDzn3OwT9jFgNjAZKAJucs5tDHy5IlIbBcXlvLRmN/M/20NZhY/rx3XnZxf0pV2Lxl6XJnWoNmfoFcAdzrmNZhYNbDCzZc65HdX2uQTo43+NBV70fxWRelRcVsmrX2Tw4sfpFBSXc+nQLtxxYV/N4xkmThnozrlsINv/vtDMdgKxQPVAvwx4zTnngC/NrI2ZdfF/r4jUsaqnIR7gl4uSyC0sZUK/GO68qB+DY1t7XZrUo9PqQzezBGAEsPaETbFAZrXlff51/xToZjYTmAkQHx9/mqWKSE2S9hfw0Ls7WLcnn8GxrZhzzUjG9NBjbcNRrQPdzFoCC4GfO+eOnsnBnHPzgHkAiYmJ7kx+hohUqaj0seDzDJ74MJnoppE8NHUQ08fE0SRS07+Fq1oFuplFURXmf3HOvV3DLllAXLXlbv51IlIH0nKPccebW9iSeYQJ/WL4/Q+H06a5LniGu9qMcjHgZWCnc+7pb9ltMXCbmb1B1cXQAvWfiwReSXklL3ycztzV6TRvHMGca0YwZWhXr8uSBqI2Z+jnAtcD28xss3/dfUA8gHNuLvA+VUMW06gatjgj4JWKhLmVu3L41eIkMvOLmTqsKw9cOoCOrZp6XZY0ILUZ5fIpcNK7EPyjW24NVFEi8v8y84t4eMkOlu3IoVdMC00BJ99Kd4qKNFDFZZXMW7ObFz5Oo5EZd0/qzy3n9aBxpKaAk5op0EUaGJ/PsXjLfh7/cBfZBSVcOqQL9186gK569oqcggJdpAHZnlXAA//YzubMIwyJbc3vfzicsT3be12WBAkFukgDsDvvGE8vS+H9bdm0a9GE3105jMtHxNJID9GS06BAF/FQSXklc1amseDzDAz40Xd6cuv5vWndLMrr0iQIKdBFPLLx68Pc9eYW0vOOM2lQZ+6/dABx7Zp7XZYEMQW6SD0rKa/kmeUpvLRmN51bNeVPt4zhO31ivC5LQoACXaSeVD0RMYcnlu5id95xrh4Tx32TBxDdVN0rEhgKdJF6kHygkPvf2cb6vYfpGdOC124ew3f76qxcAkuBLlKHqs/lGd00ksd/MIQfjOxGZIRuDpLAU6CL1IGsI8XMWZnKm+v3Ed00kjsv6su1Y7vTVlPASR1SoIsEUM7REp5flcYb66rme7l2bDy3TexDTHQTjyuTcKBAFwmAorIK5q7ezR9Wp1Ppc1yZGMdtE3sTq9v1pR4p0EXOgs/nWLItm9++v5PsghK+P6wrd13Uj/j2Gk8u9U+BLnIGfD7Hh0kHeHZFKrsOFDI4thXPXj2C0Qmay1O8o0AXOQ3OOd7fdoDZK1JIyTlGz5gWPPPDYUwdFkuEnrsiHlOgi9SCc44v0g/x1LIUNuw9TO+OLZk9fThThnZVkEuDoUAXOYX0vGPc9/Y21u7Jp0PLJjx2+RCuTIxTkEuDo0AX+RaVPscfP9nNUx+l0DSqEY9OG8wPRnajWeMIr0sTqZECXaQGKTmFzHprK5szj3DxoE48Mm0wHaM1IbM0bAp0kWoKist5bkUqr36RQXTTKGZPH87UYV0xU/eKNHwKdBGqLnou2ryfh5fs4HBRGVeNiuPuS/rTTrfqSxBRoEvY23XgKI9/sItVyXkMj2vDn24Zw6Curb0uS+S0KdAlbOUfL+Ohd5NYtHk/zaIi+OWUgdx4ToJGr0jQUqBL2Kn0ORZvyeKJD5M5dKyM287vzc3n9VD3igQ9BbqElU9TD/LwkiRSco4xOLYVc68bxbC4Nl6XJRIQCnQJC4ePl/HoeztZuHEfCe2b89zVI7h0SBcaqXtFQsgpA93M5gNTgFzn3OAatk8AFgF7/Kveds49HMAaRc5YdkExz65IZeGGLHzOcev5vfjpxD40jdLNQRJ6anOGvgCYA7x2kn0+cc5NCUhFIgFQUl7J7BWpvPzpHpxzXJUYxw3jE+jXOdrr0kTqzCkD3Tm3xswS6qEWkYDY+PVh7n5rK6m5x7h8RCy/uLAvce30fHIJfYHqQx9vZluA/cCdzrmkmnYys5nATID4+PgAHVqkyuHjZTyyZAdvb8qiY3QTFswYzYR+Hb0uS6TeBCLQNwLdnXPHzGwy8A+gT007OufmAfMAEhMTXQCOLYLP53hzQyZPfJhMQXE5t57fi59M6E3LJrrmL+HlrD/xzrmj1d6/b2YvmFkH59zBs/3ZIifjnGPxlv08vyqNlJxjjIxvw6PThjCwayuvSxPxxFkHupl1BnKcc87MxgCNgENnXZnISWQXFPPIkh28v+0A/TtH6yFaItRu2OLrwASgg5ntA34FRAE45+YCVwA/MbMKoBiY7pxTd4rUiaT9BcxdvZv3t2XTyGDWpH7853d76XZ9EWo3yuXqU2yfQ9WwRpE6k1tYwpyVafxl7dc0j4pgxjkJ3HhOgkaviFSjq0bSoBUUlfOHNem88lkGZZU+po+OY9ak/rRuFuV1aSINjgJdGqSS8krmf7aHuR+nc7SkgqnDuvKLC/vSo0MLr0sTabAU6NLgbNibz31vbyc5p5CJ/Tty50X9NHJFpBYU6NIgOOfYnHmEFz9O56MdOcREN+GVGaM5XzcGidSaAl08l5JTyAPvbGddRj4tGkdw18X9mHFuAs0b6+Mpcjr0N0Y8U1xWybMrU3lpzW5aNo3k4csG8e8jYoluqgueImdCgS6eWLUrlwcXbWff4WKuGNWN+yYP0IxBImdJgS716kBBCQ+9m8QH2w/Qu2NL3pg5jnE923tdlkhIUKBLvaio9PHaF3t56qNkKnyOuy7ux398pyeNIxt5XZpIyFCgS53bknmE+/+xje1ZR/lu3xgeuWwQ3dtrPLlIoCnQpc7kHy/jdx8l8/q6r4lp2YQ511TN46kHaInUDQW6BJzP53h3635+uSiJY6UV3HROAv99YV+NXhGpYwp0CZhKn2PJ1v9/PvmQ2NY8ddUw+nbSPJ4i9UGBLgGxNOkAj32wiz0Hj9OnY0tmTx/OpUO6EBmhi54i9UWBLmclLbeQxz7YxfKdufTrFM3c60Zy0cDONNLzyUXqnQJdzsjx0gp+91Eyr32xl2ZREdxzSX9uOa8HUTojF/GMAl1OyzcXPJ/4MJn9BcVcOzaeX1zQl/Ytm3hdmkjYU6BLraXkFHLXW1vZknmE/p2jeevq8Yzq3s7rskTET4Eup+Sc409f7uU37+2kZZNInrpyGP8+Ilb95CINjAJdTiqvsJRZb21hVXIeE/rF8OQVw4iJVveKSEOkQJdvlZZbyA0vr+Pg8TIemjqIG8Z3112eIg2YAl3+RVFZBc+tTOOPn+ymRZNIFv74HIZ0a+11WSJyCgp0+Scfbj/Aw+8msb+ghB+M7Ma9k/vTQSNYRIKCAl2AqnHlTy5NZsHnGfTvHM3sq0cwOkEjWESCiQJd2Jl9lFv/upE9B49z4/juPDhloG7ZFwlCCvQwVlBcznMrUnnti720bh7FX24Zyzm9O3hdloicIQV6GPL5HH9fn8mTS5PJLyrjipHduPsS9ZWLBLtTBrqZzQemALnOucE1bDdgNjAZKAJucs5tDHShEhj5x8u44++bWZWcx+iEtrz6/TEMjtUIFpFQUJsz9AXAHOC1b9l+CdDH/xoLvOj/Kg3MJ6l53PnmFg4fL+fhywZx/TiNKxcJJacMdOfcGjNLOMkulwGvOecc8KWZtTGzLs657EAVKWcn4+BxfvP+TpbtyKFnhxbMv2k0g7rqrFwk1ASiDz0WyKy2vM+/7l8C3cxmAjMB4uPjA3BoOZmS8kqeWZ7CK59mEBlh3HVxP245rwdNoyK8Lk1E6kC9XhR1zs0D5gEkJia6+jx2uPk4OZfHPtjFrgOF/GBkN2ZN6kenVk29LktE6lAgAj0LiKu23M2/TjxQWlHJvQu38famLOLbNWfe9aO4aFBnr8sSkXoQiEBfDNxmZm9QdTG0QP3n9a+kvJK/r8/kxY/TyS4o4faJvbltYh8aR+oGIZFwUZthi68DE4AOZrYP+BUQBeCcmwu8T9WQxTSqhi3OqKtipWYfJR3gwUXbyTlaSmL3tjx5xTDO66MbhETCTW1GuVx9iu0OuDVgFUmtFZaU8/SyFF75LIPBsa145ofDGd+zvYYiioQp3SkahJxzLNmazSNLdpB3rJQbxnfnvskDNHpFJMwp0INMZn4R9769jU/TDjI4thXzbkhkeFwbr8sSkQZAgR4knHO8vTGLXy1OAuDhywZx7djuRGheTxHxU6AHgdScQmYt3Mqmr48wJqEdT101jLh2zb0uS0QaGAV6A5aWW8gLH6ezePN+WjeL4rHLh3BlYpzOykWkRgr0Bqi0opL5n2bwzPIUIhsZ143rzn9N6EVH3ekpIiehQG9gduz//9mDLhzYid9ePkTPKReRWlGgNxA+n2P+Z3t4cmkybZs35tWbx/C9vjFelyUiQUSB3gBk5hdxx5tbWLcnnwsGdOR/Lh9Cx2h1r4jI6VGge6i4rJJ5a3Yzb006ZsYTVwzlylHddKeniJwRBbpH1u4+xN0Lt5JxqIgLBnTi11MH0q2thiKKyJlToNez46UVPLk0mQWfZxDfrjmv/8c4xvdq73VZIhICFOj15HhpBc+tTGPB53sorfAx49wE7rq4H80b649ARAJDaVLHnHO8ty2b37y3k+yCEqYN78qN5yQwIr6t16WJSIhRoNehvMJS7n9nGx/tyGFgl1bMuWYEo7q387osEQlRCvQ64JzjzfX7eGDRdnBw/+QB3HxeD92yLyJ1SoEeYNv2FfDcylQ+2pHD+J7t+c2/D6ZnTEuvyxKRMKBAD5DUnEIeXLSdL3fn07xxBLMm9eM/v9tLZ+UiUm8U6GcpM7+Ip5elsHjLfqKbRvLApQO4MjGO1s2ivC5NRMKMAv0MHSkq4/lVabz6+V7MYMY5Cfx4Qi89SEtEPKNAP02VPscrn+3h2RWpFJZWcMXIbvz3RX3p0rqZ16WJSJhToJ+GjIPHuePNLWzYe5jv9Y3h3sn96d+5lddliYgACvRaKSmv5HdLk3nl8wyaN45g9vThTB3WVQ/REpEGRYF+Cjv2H+UXf9tMck4h14yN5/aJfejcWo+2FZGGR4H+LSoqfcxekcqLH6fTtkVjFswYzYR+Hb0uS0TkWynQa3CstIKf/nUjq5LzuHxELA9OGUjbFo29LktE5KQU6CfYd7iIH726npScQh6dNpjrxnX3uiQRkVppVJudzGySmSWbWZqZ3VPD9pvMLM/MNvtfPwp8qXVvw97DTHv+M7IOF7NgxhiFuYgElVOeoZtZBPA8cCGwD/jKzBY753acsOvfnHO31UGN9WLR5izuemsrnVs15Y2ZifTuGO11SSIip6U2XS5jgDTn3G4AM3sDuAw4MdCDUqXP8cTSXfxh9W7G9GjHH64bpf5yEQlKtelyiQUyqy3v86870Q/MbKuZvWVmcTX9IDObaWbrzWx9Xl7eGZQbWAXF5cxY8BV/WL2b68bF8+dbxirMRSRo1aoPvRbeBRKcc0OBZcCrNe3knJvnnEt0ziXGxMQE6NBnpqzCx09f38TnaQf57eVDeHTaEBpHBurXISJS/2qTYFlA9TPubv51/8c5d8g5V+pf/CMwKjDl1Y2isgr+47X1rEnJ45Fpg7l6TLzXJYmInLXaBPpXQB8z62FmjYHpwOLqO5hZl2qLU4GdgSsxsI4UlXHdH9fySWoej10+RGEuIiHjlBdFnXMVZnYbsBSIAOY755LM7GFgvXNuMXC7mU0FKoB84KY6rPmMHSgo4Yb5a8k4WMQL145k0uAup/4mEZEgYc45Tw6cmJjo1q9fX2/H23PwONe/vJbDx8t46YZEzundod6OLSISKGa2wTmXWNO2sLhT9INt2dz7zjYamfH6zHEM7dbG65JERAIupAO9otLHnFVp/H55KkO7tWb29BH06NDC67JEROpEyAZ6pc/x33/fwuIt+5k2vCtPXjmMqAgNSxSR0BWSge7zOWa9tZXFW/Zz96T+/GRCL69LEhGpcyF3yuqc44FF21m4cR+/uKCvwlxEwkZIBbpzjieWJvPXtV/zkwm9uP3fentdkohIvQmZLpdKn+Ohd5N47Yu9XDM2nlkX99OcnyISVkIi0Ct9jp+9sYklW7OZ+d2e3DOpv8JcRMJOSAT6sytSWbI1m3su6c+Pv6c+cxEJT0Hfh77x68PMWZXG5SNiFeYiEtaCPtAf+2AXHaOb8OvLBnldioiIp4I60DPzi1i3J59rxsTTqmmU1+WIiHgqqAN90eaqx7JPG1HTBEoiIuElyAN9P2MS2hHXrrnXpYiIeC5oAz3naAmpuce4cGAnr0sREWkQgjbQv9x9CIDxvdp7XImISMMQtIH+RfohWjWNZECXVl6XIiLSIARtoH+5+xBjerQnopHuCBURgSAN9JyjJWQcKmJcz3ZelyIi0mAEZaBv21cAwPC4Nt4WIiLSgARloCftP4oZ6j8XEakmKAN9+/4CenRoQYsmIfFsMRGRgAjKQN+x/yiDu7b2ugwRkQYl6AI9/3gZWUeKGdRV3S0iItUFXaAn7a+6IDo4VmfoIiLVBV2gN42K4IIBHXWGLiJygqC7qjg6oR2jEzT+XETkRLU6QzezSWaWbGZpZnZPDdubmNnf/NvXmllCwCsVEZGTOmWgm1kE8DxwCTAQuNrMBp6w2y3AYedcb+AZ4PFAFyoiIidXmzP0MUCac263c64MeAO47IR9LgNe9b9/C/g3M9NDVkRE6lFtAj0WyKy2vM+/rsZ9nHMVQAHwL8+1NbOZZrbezNbn5eWdWcUiIlKjeh3l4pyb55xLdM4lxsTE1OehRURCXm0CPQuIq7bczb+uxn3MLBJoDRwKRIEiIlI7tQn0r4A+ZtbDzBoD04HFJ+yzGLjR//4KYKVzzgWuTBEROZVTjkN3zlWY2W3AUiACmO+cSzKzh4H1zrnFwMvAn8wsDcinKvRFRKQemVcn0maWB+w9w2/vABwMYDnBQG0OD2pzeDibNnd3ztV4EdKzQD8bZrbeOZfodR31SW0OD2pzeKirNgfds1xERKRmCnQRkRARrIE+z+sCPKA2hwe1OTzUSZuDsg9dRET+VbCeoYuIyAkU6CIiISLoAv1Uz2YPVmY238xyzWx7tXXtzGyZmaX6v7b1rzcze9b/O9hqZiO9q/zMmVmcma0ysx1mlmRmP/OvD9l2m1lTM1tnZlv8bX7Iv76Hfy6BNP/cAo3960NirgEzizCzTWa2xL8c0u0FMLMMM9tmZpvNbL1/XZ1+toMq0Gv5bPZgtQCYdMK6e4AVzrk+wAr/MlS1v4//NRN4sZ5qDLQK4A7n3EBgHHCr/88zlNtdCkx0zg0DhgOTzGwcVXMIPOOfU+AwVXMMQOjMNfAzYGe15VBv7zfOd84NrzbmvG4/2865oHkB44Gl1ZbvBe71uq4Ati8B2F5tORno4n/fBUj2v/8DcHVN+wXzC1gEXBgu7QaaAxuBsVTdNRjpX/9/n3OqHrkx3v8+0r+feV37abazmz+8JgJLAAvl9lZrdwbQ4YR1dfrZDqozdGr3bPZQ0sk5l+1/fwDo5H8fcr8H/3+tRwBrCfF2+7sfNgO5wDIgHTjiquYSgH9uV63mGmjgfg/MAnz+5faEdnu/4YCPzGyDmc30r6vTz3bQTRIdrpxzzsxCcoypmbUEFgI/d84drT7ZVSi22zlXCQw3szbAO0B/byuqO2Y2Bch1zm0wswkel1PfznPOZZlZR2CZme2qvrEuPtvBdoZem2ezh5IcM+sC4P+a618fMr8HM4uiKsz/4px727865NsN4Jw7AqyiqsuhjX8uAfjndgX7XAPnAlPNLIOq6SsnArMJ3fb+H+dclv9rLlX/cI+hjj/bwRbotXk2eyip/pz5G6nqY/5m/Q3+K+PjgIJq/40LGlZ1Kv4ysNM593S1TSHbbjOL8Z+ZY2bNqLpmsJOqYL/Cv9uJbQ7auQacc/c657o55xKo+vu60jl3LSHa3m+YWQszi/7mPXARsJ26/mx7feHgDC40TAZSqOp3vN/regLYrteBbKCcqv6zW6jqO1wBpALLgXb+fY2q0T7pwDYg0ev6z7DN51HVz7gV2Ox/TQ7ldgNDgU3+Nm8Hfulf3xNYB6QBbwJN/Oub+pfT/Nt7et2Gs2j7BGBJOLTX374t/lfSN1lV159t3fovIhIigq3LRUREvoUCXUQkRCjQRURChAJdRCREKNBFREKEAl1EJEQo0EVEQsT/AsaiNJvEOMwfAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# hard constraints\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import cvxpy as cp\n",
    "\n",
    "N = 1\n",
    "T = 500\n",
    "m = 20  # number of devices\n",
    "n = 5  # number of servers\n",
    "y_max = 110\n",
    "y_min = 90\n",
    "rs = 0.1\n",
    "beta = np.ones(n)\n",
    "beta[0] = 3\n",
    "beta[1] = 3\n",
    "seed = 0\n",
    "np.random.seed(seed)\n",
    "\n",
    "def optimization(m, n, wS, y, mu, BS, beta, CD):\n",
    "    x = cp.Variable((n + 1, m), nonneg=True)\n",
    "    obj = beta @ cp.inv_pos(cp.sqrt(x[1:, :] @ y + wS))\n",
    "\n",
    "    constraints = [0 <= x, x <= 1,\n",
    "                   x[1:, :] @ y <= BS,\n",
    "                   cp.sum(x, 0) == 1]\n",
    "    for i in range(m):\n",
    "        constraints.append((y[i] * mu[i, :] @ x[1:, i]) <= CD[i])\n",
    "\n",
    "    prob = cp.Problem(cp.Minimize(obj), constraints)\n",
    "    prob.solve()  # Returns the optimal value.\n",
    "    return x, prob\n",
    "\n",
    "def oracle(y, mu):\n",
    "    x, prob = optimization(m, n, wS, y, mu, BS, beta, CD)\n",
    "    return x.value, prob.value, prob.status\n",
    "\n",
    "def f(x, y):\n",
    "    return beta.dot(1/np.sqrt(x[1:, :].dot(y) + wS))\n",
    "\n",
    "def f_drop(x, y, mu, CD, BS):\n",
    "    yp = np.minimum(y, CD/ np.diag(mu @ x[1:, :]))\n",
    "    dp = np.minimum(x[1:, :].dot(yp), BS)\n",
    "    return beta.dot(1/np.sqrt(dp + wS))\n",
    "\n",
    "reg = np.zeros((N, T))\n",
    "\n",
    "#records all y, x_opt, x_t #yuhang yao\n",
    "y_N_T = np.zeros((N, T, m)) #yuhang yao\n",
    "x_opt_N_T = np.zeros((N, T, n + 1, m)) #yuhang yao\n",
    "x_t_N_T = np.zeros((N, T, n + 1, m)) #yuhang yao\n",
    "j_N_T = np.zeros((N, T, m))#yuhang yao\n",
    "BS_N = np.zeros((N, n))#yuhang yao\n",
    "\n",
    "for u in range(N):\n",
    "    wS = np.random.randint(15, 25, n)\n",
    "    \n",
    "#     BS = np.random.uniform(y_min*5, y_max*5, n)\n",
    "#     CD = np.random.uniform(y_min/2, y_max/2, m)\n",
    "    BS = np.random.uniform(y_min*10, y_max*10, n)\n",
    "    BS_N[u] = BS\n",
    "    CD = np.random.uniform(y_min*0.7, y_max*0.7, m)\n",
    "    mu = np.random.rand(m, n)\n",
    "    mu[:,0] = 0.5\n",
    "    mu[:,1] = 0.5\n",
    "    mu[:,2] = 0.8\n",
    "    mu[:,3] = 0.8\n",
    "    mu[:,4] = 0.8\n",
    "    # trace_gen = Trace(m, n, seed + u)\n",
    "    # mu = trace_gen.avg()\n",
    "    # mu = np.random.rand(m, n)\n",
    "    # mu_hat = np.zeros_like(mu)  # empirical mean\n",
    "    mu_hat = np.ones_like(mu)\n",
    "    T_ij = np.ones_like(mu)  # total number of times arm (i,j) is played\n",
    "    for t in range(T):\n",
    "        y = np.random.uniform(y_min, y_max, m).astype(int)\n",
    "        x_opt, f_opt, status = oracle(y, mu)\n",
    "        if 'optimal' not in status:\n",
    "            print('Solution infeasible 1')\n",
    "            break\n",
    "\n",
    "        rho_ij = np.sqrt(3 * np.log(t + 1) / (2 * T_ij)) * rs\n",
    "        mu_bar = np.maximum(mu_hat - rho_ij, 0) # LCB\n",
    "        x_tmp, f_tmp, status = oracle(y, mu_bar)\n",
    "        if 'optimal' not in status:\n",
    "            print('Solution infeasible 2')\n",
    "            break\n",
    "        \n",
    "        # mapping from x_tmp to x_t (new)\n",
    "        x_t = np.zeros((n + 1, m))\n",
    "        \n",
    "        for i in range(m):\n",
    "            cost = y[i] * mu[i,:].dot(x_tmp[1:, i])\n",
    "            if cost > CD[i]:\n",
    "                x_t[1:, i] = mu_bar[i] / mu[i] * x_tmp[1:, i]\n",
    "                x_t[0, i] = 1-np.sum(x_t[1:, i])\n",
    "                if x_t[0, i] < 0:\n",
    "                    x_t[0, i] = 0\n",
    "                    x_t[1:, i] = x_t[1:, i]/np.sum(x_t[1:, i])\n",
    "            else:\n",
    "                x_t[:, i] = x_tmp[:, i]\n",
    "\n",
    "        f_t = f(x_t, y)\n",
    "        \n",
    "        # sample j based on x_t[i], observe c_ij, update mu_hat[i,j]\n",
    "        # c = trace_gen.generate()\n",
    "        for i in range(m):\n",
    "            j = np.random.choice(n+1, p=x_t[:, i])\n",
    "            j_N_T[u, t, i] = j #yuhang yao\n",
    "            if j != 0:\n",
    "                j -= 1\n",
    "                c_ij = int(np.random.rand() < mu[i, j])\n",
    "                # a = np.random.rand() * 3\n",
    "                # c_ij = np.random.beta(a, a * (1-mu[i, j])/mu[i, j]) # beta distribution\n",
    "                # c_ij = c[i, j]  # trace\n",
    "                T_ij[i, j] += 1\n",
    "                mu_hat[i, j] += (c_ij - mu_hat[i, j]) / T_ij[i, j]\n",
    "\n",
    "        # calculate regert\n",
    "        reg[u, t] = f_t - f_opt\n",
    "        y_N_T[u, t] = y#yuhang yao\n",
    "        x_opt_N_T[u, t] = x_opt#yuhang yao\n",
    "        x_t_N_T[u, t] = x_t#yuhang yao\n",
    "        \n",
    "plt.plot(np.cumsum(reg, axis=1).T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "banned-calvin",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "chemical-luther",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "np.save(\"hard_wS\", wS)\n",
    "np.save(\"hard_BS_N\", BS_N)\n",
    "np.save(\"hard_y_N_T\", y_N_T)\n",
    "np.save(\"hard_x_opt_N_T\", x_opt_N_T)\n",
    "np.save(\"hard_x_t_N_T\", x_t_N_T)\n",
    "np.save(\"hard_j_N_T\", j_N_T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "applicable-watch",
   "metadata": {},
   "outputs": [],
   "source": [
    "from functions import *\n",
    "wS = np.load(\"hard_wS.npy\")\n",
    "y_N_T = np.load(\"hard_y_N_T.npy\")\n",
    "x_opt_N_T = np.load(\"hard_x_opt_N_T.npy\")\n",
    "x_t_N_T = np.load(\"hard_x_t_N_T.npy\")\n",
    "j_N_T = np.load(\"hard_j_N_T.npy\")\n",
    "BS_N = np.load(\"hard_BS_N.npy\")\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "fifth-indication",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "wS = wS.astype(int)\n",
    "BS_N = BS_N.astype(int)\n",
    "y_N_T = y_N_T.astype(int)\n",
    "\n",
    "j_N_T = j_N_T.astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2061cedb",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yuhangya/Library/Python/3.8/lib/python/site-packages/torchvision/datasets/mnist.py:498: UserWarning: The given NumPy array is not writeable, and PyTorch does not support non-writeable tensors. This means you can write to the underlying (supposedly non-writeable) NumPy array using the tensor. You may want to copy the array to protect its data or make it writeable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at  ../torch/csrc/utils/tensor_numpy.cpp:180.)\n",
      "  return torch.from_numpy(parsed.astype(m[2], copy=False)).view(*s)\n",
      "/Users/yuhangya/Library/Python/3.8/lib/python/site-packages/torchvision/datasets/mnist.py:52: UserWarning: train_labels has been renamed targets\n",
      "  warnings.warn(\"train_labels has been renamed targets\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MLP(\n",
      "  (layer_input): Linear(in_features=784, out_features=200, bias=True)\n",
      "  (relu): ReLU()\n",
      "  (dropout): Dropout(p=0.5, inplace=False)\n",
      "  (layer_hidden): Linear(in_features=200, out_features=10, bias=True)\n",
      ")\n",
      "Round   0, Average Train loss 2.113, Average Train Accuracy 30.649\n",
      "Testing accuracy: 19.76\n",
      "Testing loss: 2.22\n",
      "Round   1, Average Train loss 1.996, Average Train Accuracy 37.928\n",
      "Testing accuracy: 27.04\n",
      "Testing loss: 2.16\n",
      "Round   2, Average Train loss 1.929, Average Train Accuracy 43.572\n",
      "Testing accuracy: 24.93\n",
      "Testing loss: 2.10\n",
      "Round   3, Average Train loss 1.816, Average Train Accuracy 49.796\n",
      "Testing accuracy: 28.92\n",
      "Testing loss: 2.04\n",
      "Round   4, Average Train loss 1.737, Average Train Accuracy 52.496\n",
      "Testing accuracy: 27.99\n",
      "Testing loss: 1.99\n",
      "Round   5, Average Train loss 1.663, Average Train Accuracy 53.856\n",
      "Testing accuracy: 32.08\n",
      "Testing loss: 1.94\n",
      "Round   6, Average Train loss 1.582, Average Train Accuracy 57.320\n",
      "Testing accuracy: 33.92\n",
      "Testing loss: 1.89\n",
      "Round   7, Average Train loss 1.517, Average Train Accuracy 58.441\n",
      "Testing accuracy: 41.08\n",
      "Testing loss: 1.83\n",
      "Round   8, Average Train loss 1.472, Average Train Accuracy 61.869\n",
      "Testing accuracy: 47.72\n",
      "Testing loss: 1.76\n",
      "Round   9, Average Train loss 1.450, Average Train Accuracy 64.045\n",
      "Testing accuracy: 45.76\n",
      "Testing loss: 1.73\n",
      "Round  10, Average Train loss 1.400, Average Train Accuracy 63.341\n",
      "Testing accuracy: 49.77\n",
      "Testing loss: 1.68\n",
      "Round  11, Average Train loss 1.318, Average Train Accuracy 66.642\n",
      "Testing accuracy: 56.54\n",
      "Testing loss: 1.62\n",
      "Round  12, Average Train loss 1.281, Average Train Accuracy 69.500\n",
      "Testing accuracy: 62.87\n",
      "Testing loss: 1.56\n",
      "Round  13, Average Train loss 1.212, Average Train Accuracy 75.242\n",
      "Testing accuracy: 52.74\n",
      "Testing loss: 1.55\n",
      "Round  14, Average Train loss 1.221, Average Train Accuracy 68.270\n",
      "Testing accuracy: 61.73\n",
      "Testing loss: 1.48\n",
      "Round  15, Average Train loss 1.160, Average Train Accuracy 73.346\n",
      "Testing accuracy: 62.10\n",
      "Testing loss: 1.44\n",
      "Round  16, Average Train loss 1.131, Average Train Accuracy 73.723\n",
      "Testing accuracy: 63.70\n",
      "Testing loss: 1.40\n",
      "Round  17, Average Train loss 1.111, Average Train Accuracy 73.717\n",
      "Testing accuracy: 65.03\n",
      "Testing loss: 1.36\n",
      "Round  18, Average Train loss 1.063, Average Train Accuracy 75.742\n",
      "Testing accuracy: 66.33\n",
      "Testing loss: 1.32\n",
      "Round  19, Average Train loss 1.047, Average Train Accuracy 75.624\n",
      "Testing accuracy: 66.55\n",
      "Testing loss: 1.31\n",
      "Round  20, Average Train loss 1.030, Average Train Accuracy 75.178\n",
      "Testing accuracy: 67.38\n",
      "Testing loss: 1.27\n",
      "Round  21, Average Train loss 0.989, Average Train Accuracy 76.153\n",
      "Testing accuracy: 71.68\n",
      "Testing loss: 1.22\n",
      "Round  22, Average Train loss 0.957, Average Train Accuracy 78.180\n",
      "Testing accuracy: 72.63\n",
      "Testing loss: 1.19\n",
      "Round  23, Average Train loss 0.947, Average Train Accuracy 78.726\n",
      "Testing accuracy: 72.72\n",
      "Testing loss: 1.16\n",
      "Round  24, Average Train loss 0.918, Average Train Accuracy 79.468\n",
      "Testing accuracy: 71.59\n",
      "Testing loss: 1.15\n",
      "Round  25, Average Train loss 0.878, Average Train Accuracy 79.643\n",
      "Testing accuracy: 70.85\n",
      "Testing loss: 1.14\n",
      "Round  26, Average Train loss 0.881, Average Train Accuracy 78.624\n",
      "Testing accuracy: 70.56\n",
      "Testing loss: 1.11\n",
      "Round  27, Average Train loss 0.850, Average Train Accuracy 80.358\n",
      "Testing accuracy: 70.44\n",
      "Testing loss: 1.07\n",
      "Round  28, Average Train loss 0.859, Average Train Accuracy 78.840\n",
      "Testing accuracy: 74.14\n",
      "Testing loss: 1.02\n",
      "Round  29, Average Train loss 0.844, Average Train Accuracy 80.063\n",
      "Testing accuracy: 75.32\n",
      "Testing loss: 0.99\n",
      "Round  30, Average Train loss 0.820, Average Train Accuracy 80.584\n",
      "Testing accuracy: 74.37\n",
      "Testing loss: 0.97\n",
      "Round  31, Average Train loss 0.828, Average Train Accuracy 79.528\n",
      "Testing accuracy: 75.14\n",
      "Testing loss: 0.96\n",
      "Round  32, Average Train loss 0.769, Average Train Accuracy 81.457\n",
      "Testing accuracy: 75.82\n",
      "Testing loss: 0.94\n",
      "Round  33, Average Train loss 0.776, Average Train Accuracy 81.381\n",
      "Testing accuracy: 75.94\n",
      "Testing loss: 0.92\n",
      "Round  34, Average Train loss 0.747, Average Train Accuracy 82.367\n",
      "Testing accuracy: 76.43\n",
      "Testing loss: 0.90\n",
      "Round  35, Average Train loss 0.746, Average Train Accuracy 82.485\n",
      "Testing accuracy: 76.73\n",
      "Testing loss: 0.88\n",
      "Round  36, Average Train loss 0.762, Average Train Accuracy 81.479\n",
      "Testing accuracy: 76.88\n",
      "Testing loss: 0.87\n",
      "Round  37, Average Train loss 0.753, Average Train Accuracy 81.669\n",
      "Testing accuracy: 77.58\n",
      "Testing loss: 0.85\n",
      "Round  38, Average Train loss 0.709, Average Train Accuracy 82.464\n",
      "Testing accuracy: 77.03\n",
      "Testing loss: 0.85\n",
      "Round  39, Average Train loss 0.722, Average Train Accuracy 83.173\n",
      "Testing accuracy: 77.16\n",
      "Testing loss: 0.84\n",
      "Round  40, Average Train loss 0.714, Average Train Accuracy 82.624\n",
      "Testing accuracy: 77.38\n",
      "Testing loss: 0.82\n",
      "Round  41, Average Train loss 0.706, Average Train Accuracy 82.318\n",
      "Testing accuracy: 77.82\n",
      "Testing loss: 0.80\n",
      "Round  42, Average Train loss 0.690, Average Train Accuracy 82.700\n",
      "Testing accuracy: 77.36\n",
      "Testing loss: 0.81\n",
      "Round  43, Average Train loss 0.687, Average Train Accuracy 82.404\n",
      "Testing accuracy: 78.01\n",
      "Testing loss: 0.78\n",
      "Round  44, Average Train loss 0.672, Average Train Accuracy 81.834\n",
      "Testing accuracy: 78.94\n",
      "Testing loss: 0.76\n",
      "Round  45, Average Train loss 0.682, Average Train Accuracy 82.688\n",
      "Testing accuracy: 78.81\n",
      "Testing loss: 0.75\n",
      "Round  46, Average Train loss 0.672, Average Train Accuracy 82.880\n",
      "Testing accuracy: 78.67\n",
      "Testing loss: 0.75\n",
      "Round  47, Average Train loss 0.669, Average Train Accuracy 82.579\n",
      "Testing accuracy: 79.57\n",
      "Testing loss: 0.73\n",
      "Round  48, Average Train loss 0.666, Average Train Accuracy 82.490\n",
      "Testing accuracy: 79.46\n",
      "Testing loss: 0.72\n",
      "Round  49, Average Train loss 0.638, Average Train Accuracy 84.285\n",
      "Testing accuracy: 79.51\n",
      "Testing loss: 0.71\n",
      "Round  50, Average Train loss 0.631, Average Train Accuracy 83.668\n",
      "Testing accuracy: 78.95\n",
      "Testing loss: 0.72\n",
      "Round  51, Average Train loss 0.628, Average Train Accuracy 84.549\n",
      "Testing accuracy: 80.04\n",
      "Testing loss: 0.70\n",
      "Round  52, Average Train loss 0.620, Average Train Accuracy 84.044\n",
      "Testing accuracy: 79.75\n",
      "Testing loss: 0.69\n",
      "Round  53, Average Train loss 0.611, Average Train Accuracy 84.222\n",
      "Testing accuracy: 79.39\n",
      "Testing loss: 0.70\n",
      "Round  54, Average Train loss 0.622, Average Train Accuracy 83.335\n",
      "Testing accuracy: 79.28\n",
      "Testing loss: 0.69\n",
      "Round  55, Average Train loss 0.582, Average Train Accuracy 84.827\n",
      "Testing accuracy: 79.94\n",
      "Testing loss: 0.68\n",
      "Round  56, Average Train loss 0.580, Average Train Accuracy 84.433\n",
      "Testing accuracy: 78.16\n",
      "Testing loss: 0.70\n",
      "Round  57, Average Train loss 0.597, Average Train Accuracy 83.910\n",
      "Testing accuracy: 80.41\n",
      "Testing loss: 0.67\n",
      "Round  58, Average Train loss 0.600, Average Train Accuracy 84.467\n",
      "Testing accuracy: 81.76\n",
      "Testing loss: 0.64\n",
      "Round  59, Average Train loss 0.587, Average Train Accuracy 84.712\n",
      "Testing accuracy: 80.78\n",
      "Testing loss: 0.65\n",
      "Round  60, Average Train loss 0.597, Average Train Accuracy 83.795\n",
      "Testing accuracy: 80.48\n",
      "Testing loss: 0.65\n",
      "Round  61, Average Train loss 0.565, Average Train Accuracy 84.921\n",
      "Testing accuracy: 81.45\n",
      "Testing loss: 0.63\n",
      "Round  62, Average Train loss 0.550, Average Train Accuracy 86.259\n",
      "Testing accuracy: 81.69\n",
      "Testing loss: 0.62\n",
      "Round  63, Average Train loss 0.577, Average Train Accuracy 84.394\n",
      "Testing accuracy: 81.43\n",
      "Testing loss: 0.63\n",
      "Round  64, Average Train loss 0.578, Average Train Accuracy 84.220\n",
      "Testing accuracy: 80.77\n",
      "Testing loss: 0.63\n",
      "Round  65, Average Train loss 0.564, Average Train Accuracy 84.347\n",
      "Testing accuracy: 80.50\n",
      "Testing loss: 0.63\n",
      "Round  66, Average Train loss 0.569, Average Train Accuracy 84.281\n",
      "Testing accuracy: 81.09\n",
      "Testing loss: 0.62\n",
      "Round  67, Average Train loss 0.528, Average Train Accuracy 86.410\n",
      "Testing accuracy: 81.36\n",
      "Testing loss: 0.62\n",
      "Round  68, Average Train loss 0.568, Average Train Accuracy 83.554\n",
      "Testing accuracy: 82.28\n",
      "Testing loss: 0.60\n",
      "Round  69, Average Train loss 0.572, Average Train Accuracy 84.300\n",
      "Testing accuracy: 80.06\n",
      "Testing loss: 0.63\n",
      "Round  70, Average Train loss 0.525, Average Train Accuracy 85.487\n",
      "Testing accuracy: 80.83\n",
      "Testing loss: 0.61\n",
      "Round  71, Average Train loss 0.504, Average Train Accuracy 87.709\n",
      "Testing accuracy: 82.86\n",
      "Testing loss: 0.59\n",
      "Round  72, Average Train loss 0.560, Average Train Accuracy 84.263\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing accuracy: 82.84\n",
      "Testing loss: 0.59\n",
      "Round  73, Average Train loss 0.554, Average Train Accuracy 85.137\n",
      "Testing accuracy: 81.10\n",
      "Testing loss: 0.60\n",
      "Round  74, Average Train loss 0.525, Average Train Accuracy 85.509\n",
      "Testing accuracy: 81.40\n",
      "Testing loss: 0.59\n",
      "Round  75, Average Train loss 0.540, Average Train Accuracy 85.244\n",
      "Testing accuracy: 81.25\n",
      "Testing loss: 0.59\n",
      "Round  76, Average Train loss 0.512, Average Train Accuracy 85.957\n",
      "Testing accuracy: 82.20\n",
      "Testing loss: 0.58\n",
      "Round  77, Average Train loss 0.551, Average Train Accuracy 84.923\n",
      "Testing accuracy: 82.95\n",
      "Testing loss: 0.57\n",
      "Round  78, Average Train loss 0.523, Average Train Accuracy 85.934\n",
      "Testing accuracy: 83.50\n",
      "Testing loss: 0.56\n",
      "Round  79, Average Train loss 0.532, Average Train Accuracy 85.437\n",
      "Testing accuracy: 84.88\n",
      "Testing loss: 0.55\n",
      "Round  80, Average Train loss 0.535, Average Train Accuracy 85.336\n",
      "Testing accuracy: 85.16\n",
      "Testing loss: 0.55\n",
      "Round  81, Average Train loss 0.510, Average Train Accuracy 87.245\n",
      "Testing accuracy: 84.87\n",
      "Testing loss: 0.55\n",
      "Round  82, Average Train loss 0.469, Average Train Accuracy 87.550\n",
      "Testing accuracy: 85.43\n",
      "Testing loss: 0.54\n",
      "Round  83, Average Train loss 0.544, Average Train Accuracy 85.005\n",
      "Testing accuracy: 84.08\n",
      "Testing loss: 0.54\n",
      "Round  84, Average Train loss 0.533, Average Train Accuracy 85.660\n",
      "Testing accuracy: 84.11\n",
      "Testing loss: 0.54\n",
      "Round  85, Average Train loss 0.515, Average Train Accuracy 85.833\n",
      "Testing accuracy: 84.58\n",
      "Testing loss: 0.54\n",
      "Round  86, Average Train loss 0.519, Average Train Accuracy 85.717\n",
      "Testing accuracy: 84.83\n",
      "Testing loss: 0.53\n",
      "Round  87, Average Train loss 0.481, Average Train Accuracy 87.025\n",
      "Testing accuracy: 82.49\n",
      "Testing loss: 0.55\n",
      "Round  88, Average Train loss 0.495, Average Train Accuracy 86.129\n",
      "Testing accuracy: 83.28\n",
      "Testing loss: 0.54\n",
      "Round  89, Average Train loss 0.467, Average Train Accuracy 86.828\n",
      "Testing accuracy: 81.60\n",
      "Testing loss: 0.56\n",
      "Round  90, Average Train loss 0.529, Average Train Accuracy 84.663\n",
      "Testing accuracy: 83.01\n",
      "Testing loss: 0.54\n",
      "Round  91, Average Train loss 0.466, Average Train Accuracy 86.393\n",
      "Testing accuracy: 82.93\n",
      "Testing loss: 0.54\n",
      "Round  92, Average Train loss 0.485, Average Train Accuracy 86.112\n",
      "Testing accuracy: 84.08\n",
      "Testing loss: 0.52\n",
      "Round  93, Average Train loss 0.489, Average Train Accuracy 85.586\n",
      "Testing accuracy: 83.10\n",
      "Testing loss: 0.53\n",
      "Round  94, Average Train loss 0.480, Average Train Accuracy 86.675\n",
      "Testing accuracy: 84.35\n",
      "Testing loss: 0.52\n",
      "Round  95, Average Train loss 0.468, Average Train Accuracy 86.626\n",
      "Testing accuracy: 82.20\n",
      "Testing loss: 0.55\n",
      "Round  96, Average Train loss 0.499, Average Train Accuracy 85.428\n",
      "Testing accuracy: 83.85\n",
      "Testing loss: 0.52\n",
      "Round  97, Average Train loss 0.458, Average Train Accuracy 87.416\n",
      "Testing accuracy: 84.71\n",
      "Testing loss: 0.52\n",
      "Round  98, Average Train loss 0.484, Average Train Accuracy 85.693\n",
      "Testing accuracy: 84.40\n",
      "Testing loss: 0.52\n",
      "Round  99, Average Train loss 0.445, Average Train Accuracy 87.931\n",
      "Testing accuracy: 83.99\n",
      "Testing loss: 0.53\n",
      "Round 100, Average Train loss 0.496, Average Train Accuracy 85.532\n",
      "Testing accuracy: 85.33\n",
      "Testing loss: 0.51\n",
      "Round 101, Average Train loss 0.472, Average Train Accuracy 86.520\n",
      "Testing accuracy: 86.19\n",
      "Testing loss: 0.50\n",
      "Round 102, Average Train loss 0.478, Average Train Accuracy 87.088\n",
      "Testing accuracy: 85.47\n",
      "Testing loss: 0.50\n",
      "Round 103, Average Train loss 0.467, Average Train Accuracy 86.812\n",
      "Testing accuracy: 85.86\n",
      "Testing loss: 0.49\n",
      "Round 104, Average Train loss 0.472, Average Train Accuracy 86.312\n",
      "Testing accuracy: 85.65\n",
      "Testing loss: 0.49\n",
      "Round 105, Average Train loss 0.463, Average Train Accuracy 87.027\n",
      "Testing accuracy: 85.27\n",
      "Testing loss: 0.50\n",
      "Round 106, Average Train loss 0.463, Average Train Accuracy 87.084\n",
      "Testing accuracy: 85.01\n",
      "Testing loss: 0.50\n",
      "Round 107, Average Train loss 0.471, Average Train Accuracy 86.833\n",
      "Testing accuracy: 85.95\n",
      "Testing loss: 0.49\n",
      "Round 108, Average Train loss 0.469, Average Train Accuracy 86.930\n",
      "Testing accuracy: 85.41\n",
      "Testing loss: 0.49\n",
      "Round 109, Average Train loss 0.464, Average Train Accuracy 86.509\n",
      "Testing accuracy: 85.56\n",
      "Testing loss: 0.49\n",
      "Round 110, Average Train loss 0.473, Average Train Accuracy 86.110\n",
      "Testing accuracy: 85.45\n",
      "Testing loss: 0.49\n",
      "Round 111, Average Train loss 0.454, Average Train Accuracy 87.512\n",
      "Testing accuracy: 84.99\n",
      "Testing loss: 0.50\n",
      "Round 112, Average Train loss 0.445, Average Train Accuracy 86.917\n",
      "Testing accuracy: 86.07\n",
      "Testing loss: 0.48\n",
      "Round 113, Average Train loss 0.445, Average Train Accuracy 87.568\n",
      "Testing accuracy: 86.15\n",
      "Testing loss: 0.48\n",
      "Round 114, Average Train loss 0.435, Average Train Accuracy 87.656\n",
      "Testing accuracy: 85.37\n",
      "Testing loss: 0.49\n",
      "Round 115, Average Train loss 0.440, Average Train Accuracy 87.574\n",
      "Testing accuracy: 85.20\n",
      "Testing loss: 0.49\n",
      "Round 116, Average Train loss 0.427, Average Train Accuracy 86.775\n",
      "Testing accuracy: 85.73\n",
      "Testing loss: 0.48\n",
      "Round 117, Average Train loss 0.440, Average Train Accuracy 87.499\n",
      "Testing accuracy: 85.40\n",
      "Testing loss: 0.48\n",
      "Round 118, Average Train loss 0.430, Average Train Accuracy 87.682\n",
      "Testing accuracy: 85.56\n",
      "Testing loss: 0.48\n",
      "Round 119, Average Train loss 0.415, Average Train Accuracy 88.492\n",
      "Testing accuracy: 86.28\n",
      "Testing loss: 0.47\n",
      "Round 120, Average Train loss 0.467, Average Train Accuracy 86.686\n",
      "Testing accuracy: 84.71\n",
      "Testing loss: 0.49\n",
      "Round 121, Average Train loss 0.428, Average Train Accuracy 87.605\n",
      "Testing accuracy: 84.51\n",
      "Testing loss: 0.49\n",
      "Round 122, Average Train loss 0.444, Average Train Accuracy 86.991\n",
      "Testing accuracy: 83.90\n",
      "Testing loss: 0.50\n",
      "Round 123, Average Train loss 0.440, Average Train Accuracy 87.009\n",
      "Testing accuracy: 84.65\n",
      "Testing loss: 0.49\n",
      "Round 124, Average Train loss 0.408, Average Train Accuracy 88.242\n",
      "Testing accuracy: 85.80\n",
      "Testing loss: 0.47\n",
      "Round 125, Average Train loss 0.442, Average Train Accuracy 87.606\n",
      "Testing accuracy: 86.11\n",
      "Testing loss: 0.47\n",
      "Round 126, Average Train loss 0.439, Average Train Accuracy 86.860\n",
      "Testing accuracy: 85.62\n",
      "Testing loss: 0.47\n",
      "Round 127, Average Train loss 0.444, Average Train Accuracy 86.948\n",
      "Testing accuracy: 86.49\n",
      "Testing loss: 0.46\n",
      "Round 128, Average Train loss 0.444, Average Train Accuracy 87.323\n",
      "Testing accuracy: 86.75\n",
      "Testing loss: 0.45\n",
      "Round 129, Average Train loss 0.428, Average Train Accuracy 87.606\n",
      "Testing accuracy: 86.56\n",
      "Testing loss: 0.46\n",
      "Round 130, Average Train loss 0.439, Average Train Accuracy 86.881\n",
      "Testing accuracy: 87.36\n",
      "Testing loss: 0.44\n",
      "Round 131, Average Train loss 0.446, Average Train Accuracy 87.306\n",
      "Testing accuracy: 86.76\n",
      "Testing loss: 0.45\n",
      "Round 132, Average Train loss 0.445, Average Train Accuracy 86.902\n",
      "Testing accuracy: 86.85\n",
      "Testing loss: 0.45\n",
      "Round 133, Average Train loss 0.429, Average Train Accuracy 87.827\n",
      "Testing accuracy: 87.27\n",
      "Testing loss: 0.44\n",
      "Round 134, Average Train loss 0.446, Average Train Accuracy 87.414\n",
      "Testing accuracy: 86.97\n",
      "Testing loss: 0.45\n",
      "Round 135, Average Train loss 0.442, Average Train Accuracy 87.548\n",
      "Testing accuracy: 86.95\n",
      "Testing loss: 0.45\n",
      "Round 136, Average Train loss 0.433, Average Train Accuracy 87.123\n",
      "Testing accuracy: 87.09\n",
      "Testing loss: 0.44\n",
      "Round 137, Average Train loss 0.417, Average Train Accuracy 88.499\n",
      "Testing accuracy: 86.40\n",
      "Testing loss: 0.46\n",
      "Round 138, Average Train loss 0.416, Average Train Accuracy 88.253\n",
      "Testing accuracy: 87.06\n",
      "Testing loss: 0.44\n",
      "Round 139, Average Train loss 0.426, Average Train Accuracy 87.482\n",
      "Testing accuracy: 87.67\n",
      "Testing loss: 0.43\n",
      "Round 140, Average Train loss 0.434, Average Train Accuracy 87.358\n",
      "Testing accuracy: 86.77\n",
      "Testing loss: 0.44\n",
      "Round 141, Average Train loss 0.380, Average Train Accuracy 88.691\n",
      "Testing accuracy: 87.33\n",
      "Testing loss: 0.44\n",
      "Round 142, Average Train loss 0.403, Average Train Accuracy 88.877\n",
      "Testing accuracy: 86.75\n",
      "Testing loss: 0.44\n",
      "Round 143, Average Train loss 0.414, Average Train Accuracy 87.661\n",
      "Testing accuracy: 87.23\n",
      "Testing loss: 0.43\n",
      "Round 144, Average Train loss 0.449, Average Train Accuracy 86.885\n",
      "Testing accuracy: 87.68\n",
      "Testing loss: 0.43\n",
      "Round 145, Average Train loss 0.428, Average Train Accuracy 87.590\n",
      "Testing accuracy: 87.41\n",
      "Testing loss: 0.43\n",
      "Round 146, Average Train loss 0.460, Average Train Accuracy 86.217\n",
      "Testing accuracy: 86.87\n",
      "Testing loss: 0.44\n",
      "Round 147, Average Train loss 0.411, Average Train Accuracy 88.151\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing accuracy: 84.98\n",
      "Testing loss: 0.47\n",
      "Round 148, Average Train loss 0.410, Average Train Accuracy 88.468\n",
      "Testing accuracy: 86.88\n",
      "Testing loss: 0.44\n",
      "Round 149, Average Train loss 0.400, Average Train Accuracy 89.246\n",
      "Testing accuracy: 86.71\n",
      "Testing loss: 0.45\n",
      "Round 150, Average Train loss 0.421, Average Train Accuracy 88.016\n",
      "Testing accuracy: 85.95\n",
      "Testing loss: 0.46\n",
      "Round 151, Average Train loss 0.427, Average Train Accuracy 88.013\n",
      "Testing accuracy: 86.54\n",
      "Testing loss: 0.45\n",
      "Round 152, Average Train loss 0.392, Average Train Accuracy 88.469\n",
      "Testing accuracy: 87.36\n",
      "Testing loss: 0.43\n",
      "Round 153, Average Train loss 0.401, Average Train Accuracy 88.695\n",
      "Testing accuracy: 87.46\n",
      "Testing loss: 0.43\n",
      "Round 154, Average Train loss 0.420, Average Train Accuracy 87.519\n",
      "Testing accuracy: 87.07\n",
      "Testing loss: 0.43\n",
      "Round 155, Average Train loss 0.381, Average Train Accuracy 88.709\n",
      "Testing accuracy: 86.91\n",
      "Testing loss: 0.44\n",
      "Round 156, Average Train loss 0.379, Average Train Accuracy 89.334\n",
      "Testing accuracy: 87.41\n",
      "Testing loss: 0.43\n",
      "Round 157, Average Train loss 0.439, Average Train Accuracy 87.544\n",
      "Testing accuracy: 87.06\n",
      "Testing loss: 0.43\n",
      "Round 158, Average Train loss 0.406, Average Train Accuracy 88.062\n",
      "Testing accuracy: 86.73\n",
      "Testing loss: 0.44\n",
      "Round 159, Average Train loss 0.407, Average Train Accuracy 88.856\n",
      "Testing accuracy: 86.62\n",
      "Testing loss: 0.44\n",
      "Round 160, Average Train loss 0.372, Average Train Accuracy 90.006\n",
      "Testing accuracy: 88.11\n",
      "Testing loss: 0.42\n",
      "Round 161, Average Train loss 0.394, Average Train Accuracy 89.395\n",
      "Testing accuracy: 87.50\n",
      "Testing loss: 0.43\n",
      "Round 162, Average Train loss 0.406, Average Train Accuracy 88.816\n",
      "Testing accuracy: 87.75\n",
      "Testing loss: 0.42\n",
      "Round 163, Average Train loss 0.394, Average Train Accuracy 88.416\n",
      "Testing accuracy: 87.34\n",
      "Testing loss: 0.43\n",
      "Round 164, Average Train loss 0.399, Average Train Accuracy 88.329\n",
      "Testing accuracy: 87.98\n",
      "Testing loss: 0.41\n",
      "Round 165, Average Train loss 0.398, Average Train Accuracy 88.642\n",
      "Testing accuracy: 87.95\n",
      "Testing loss: 0.41\n",
      "Round 166, Average Train loss 0.381, Average Train Accuracy 88.839\n",
      "Testing accuracy: 87.86\n",
      "Testing loss: 0.41\n",
      "Round 167, Average Train loss 0.384, Average Train Accuracy 88.902\n",
      "Testing accuracy: 88.06\n",
      "Testing loss: 0.41\n",
      "Round 168, Average Train loss 0.411, Average Train Accuracy 88.545\n",
      "Testing accuracy: 87.01\n",
      "Testing loss: 0.43\n",
      "Round 169, Average Train loss 0.378, Average Train Accuracy 89.410\n",
      "Testing accuracy: 88.02\n",
      "Testing loss: 0.41\n",
      "Round 170, Average Train loss 0.389, Average Train Accuracy 89.079\n",
      "Testing accuracy: 87.91\n",
      "Testing loss: 0.41\n",
      "Round 171, Average Train loss 0.396, Average Train Accuracy 89.148\n",
      "Testing accuracy: 87.79\n",
      "Testing loss: 0.41\n",
      "Round 172, Average Train loss 0.392, Average Train Accuracy 88.845\n",
      "Testing accuracy: 88.20\n",
      "Testing loss: 0.41\n",
      "Round 173, Average Train loss 0.393, Average Train Accuracy 88.899\n",
      "Testing accuracy: 88.41\n",
      "Testing loss: 0.40\n",
      "Round 174, Average Train loss 0.395, Average Train Accuracy 88.955\n",
      "Testing accuracy: 87.55\n",
      "Testing loss: 0.42\n",
      "Round 175, Average Train loss 0.365, Average Train Accuracy 89.910\n",
      "Testing accuracy: 87.30\n",
      "Testing loss: 0.42\n",
      "Round 176, Average Train loss 0.373, Average Train Accuracy 89.195\n",
      "Testing accuracy: 87.03\n",
      "Testing loss: 0.43\n",
      "Round 177, Average Train loss 0.376, Average Train Accuracy 88.602\n",
      "Testing accuracy: 87.69\n",
      "Testing loss: 0.42\n",
      "Round 178, Average Train loss 0.355, Average Train Accuracy 90.378\n",
      "Testing accuracy: 88.49\n",
      "Testing loss: 0.40\n",
      "Round 179, Average Train loss 0.382, Average Train Accuracy 89.082\n",
      "Testing accuracy: 88.26\n",
      "Testing loss: 0.41\n",
      "Round 180, Average Train loss 0.414, Average Train Accuracy 87.382\n",
      "Testing accuracy: 88.30\n",
      "Testing loss: 0.40\n",
      "Round 181, Average Train loss 0.398, Average Train Accuracy 88.209\n",
      "Testing accuracy: 88.52\n",
      "Testing loss: 0.40\n",
      "Round 182, Average Train loss 0.394, Average Train Accuracy 88.342\n",
      "Testing accuracy: 88.59\n",
      "Testing loss: 0.40\n",
      "Round 183, Average Train loss 0.384, Average Train Accuracy 88.676\n",
      "Testing accuracy: 88.13\n",
      "Testing loss: 0.40\n",
      "Round 184, Average Train loss 0.380, Average Train Accuracy 88.900\n",
      "Testing accuracy: 88.45\n",
      "Testing loss: 0.40\n",
      "Round 185, Average Train loss 0.384, Average Train Accuracy 88.779\n",
      "Testing accuracy: 87.77\n",
      "Testing loss: 0.41\n",
      "Round 186, Average Train loss 0.363, Average Train Accuracy 89.402\n",
      "Testing accuracy: 87.95\n",
      "Testing loss: 0.41\n",
      "Round 187, Average Train loss 0.388, Average Train Accuracy 88.596\n",
      "Testing accuracy: 88.06\n",
      "Testing loss: 0.40\n",
      "Round 188, Average Train loss 0.355, Average Train Accuracy 90.240\n",
      "Testing accuracy: 88.42\n",
      "Testing loss: 0.40\n",
      "Round 189, Average Train loss 0.387, Average Train Accuracy 88.952\n",
      "Testing accuracy: 88.83\n",
      "Testing loss: 0.39\n",
      "Round 190, Average Train loss 0.374, Average Train Accuracy 90.041\n",
      "Testing accuracy: 88.83\n",
      "Testing loss: 0.39\n",
      "Round 191, Average Train loss 0.382, Average Train Accuracy 88.728\n",
      "Testing accuracy: 88.28\n",
      "Testing loss: 0.40\n",
      "Round 192, Average Train loss 0.344, Average Train Accuracy 90.175\n",
      "Testing accuracy: 88.21\n",
      "Testing loss: 0.40\n",
      "Round 193, Average Train loss 0.382, Average Train Accuracy 89.248\n",
      "Testing accuracy: 88.43\n",
      "Testing loss: 0.39\n",
      "Round 194, Average Train loss 0.339, Average Train Accuracy 90.908\n",
      "Testing accuracy: 88.08\n",
      "Testing loss: 0.40\n",
      "Round 195, Average Train loss 0.359, Average Train Accuracy 89.295\n",
      "Testing accuracy: 88.70\n",
      "Testing loss: 0.39\n",
      "Round 196, Average Train loss 0.372, Average Train Accuracy 89.522\n",
      "Testing accuracy: 87.76\n",
      "Testing loss: 0.40\n",
      "Round 197, Average Train loss 0.356, Average Train Accuracy 89.398\n",
      "Testing accuracy: 88.66\n",
      "Testing loss: 0.39\n",
      "Round 198, Average Train loss 0.342, Average Train Accuracy 90.371\n",
      "Testing accuracy: 88.96\n",
      "Testing loss: 0.38\n",
      "Round 199, Average Train loss 0.367, Average Train Accuracy 89.333\n",
      "Testing accuracy: 88.84\n",
      "Testing loss: 0.39\n",
      "Round 200, Average Train loss 0.384, Average Train Accuracy 88.620\n",
      "Testing accuracy: 88.80\n",
      "Testing loss: 0.39\n",
      "Round 201, Average Train loss 0.360, Average Train Accuracy 89.824\n",
      "Testing accuracy: 88.70\n",
      "Testing loss: 0.39\n",
      "Round 202, Average Train loss 0.358, Average Train Accuracy 88.989\n",
      "Testing accuracy: 89.22\n",
      "Testing loss: 0.38\n",
      "Round 203, Average Train loss 0.345, Average Train Accuracy 90.686\n",
      "Testing accuracy: 88.66\n",
      "Testing loss: 0.38\n",
      "Round 204, Average Train loss 0.374, Average Train Accuracy 89.621\n",
      "Testing accuracy: 88.51\n",
      "Testing loss: 0.39\n",
      "Round 205, Average Train loss 0.380, Average Train Accuracy 89.401\n",
      "Testing accuracy: 88.52\n",
      "Testing loss: 0.39\n",
      "Round 206, Average Train loss 0.361, Average Train Accuracy 89.726\n",
      "Testing accuracy: 88.60\n",
      "Testing loss: 0.39\n",
      "Round 207, Average Train loss 0.356, Average Train Accuracy 90.100\n",
      "Testing accuracy: 88.54\n",
      "Testing loss: 0.39\n",
      "Round 208, Average Train loss 0.368, Average Train Accuracy 89.522\n",
      "Testing accuracy: 88.54\n",
      "Testing loss: 0.38\n",
      "Round 209, Average Train loss 0.374, Average Train Accuracy 88.678\n",
      "Testing accuracy: 88.09\n",
      "Testing loss: 0.39\n",
      "Round 210, Average Train loss 0.355, Average Train Accuracy 88.986\n",
      "Testing accuracy: 88.16\n",
      "Testing loss: 0.39\n",
      "Round 211, Average Train loss 0.349, Average Train Accuracy 90.074\n",
      "Testing accuracy: 88.44\n",
      "Testing loss: 0.39\n",
      "Round 212, Average Train loss 0.363, Average Train Accuracy 89.538\n",
      "Testing accuracy: 87.50\n",
      "Testing loss: 0.40\n",
      "Round 213, Average Train loss 0.354, Average Train Accuracy 89.712\n",
      "Testing accuracy: 87.39\n",
      "Testing loss: 0.41\n",
      "Round 214, Average Train loss 0.352, Average Train Accuracy 89.616\n",
      "Testing accuracy: 87.53\n",
      "Testing loss: 0.40\n",
      "Round 215, Average Train loss 0.335, Average Train Accuracy 89.919\n",
      "Testing accuracy: 88.34\n",
      "Testing loss: 0.38\n",
      "Round 216, Average Train loss 0.346, Average Train Accuracy 89.863\n",
      "Testing accuracy: 88.16\n",
      "Testing loss: 0.39\n",
      "Round 217, Average Train loss 0.368, Average Train Accuracy 89.398\n",
      "Testing accuracy: 88.53\n",
      "Testing loss: 0.38\n",
      "Round 218, Average Train loss 0.336, Average Train Accuracy 90.450\n",
      "Testing accuracy: 88.88\n",
      "Testing loss: 0.37\n",
      "Round 219, Average Train loss 0.368, Average Train Accuracy 89.409\n",
      "Testing accuracy: 88.46\n",
      "Testing loss: 0.38\n",
      "Round 220, Average Train loss 0.338, Average Train Accuracy 89.905\n",
      "Testing accuracy: 88.79\n",
      "Testing loss: 0.38\n",
      "Round 221, Average Train loss 0.334, Average Train Accuracy 90.251\n",
      "Testing accuracy: 88.67\n",
      "Testing loss: 0.38\n",
      "Round 222, Average Train loss 0.355, Average Train Accuracy 89.366\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing accuracy: 88.57\n",
      "Testing loss: 0.38\n",
      "Round 223, Average Train loss 0.355, Average Train Accuracy 89.148\n",
      "Testing accuracy: 88.86\n",
      "Testing loss: 0.38\n",
      "Round 224, Average Train loss 0.345, Average Train Accuracy 90.314\n",
      "Testing accuracy: 88.49\n",
      "Testing loss: 0.38\n",
      "Round 225, Average Train loss 0.342, Average Train Accuracy 89.787\n",
      "Testing accuracy: 88.68\n",
      "Testing loss: 0.38\n",
      "Round 226, Average Train loss 0.364, Average Train Accuracy 89.126\n",
      "Testing accuracy: 88.80\n",
      "Testing loss: 0.38\n",
      "Round 227, Average Train loss 0.363, Average Train Accuracy 88.939\n",
      "Testing accuracy: 88.90\n",
      "Testing loss: 0.37\n",
      "Round 228, Average Train loss 0.332, Average Train Accuracy 90.987\n",
      "Testing accuracy: 89.14\n",
      "Testing loss: 0.37\n",
      "Round 229, Average Train loss 0.353, Average Train Accuracy 89.986\n",
      "Testing accuracy: 88.91\n",
      "Testing loss: 0.37\n",
      "Round 230, Average Train loss 0.353, Average Train Accuracy 89.887\n",
      "Testing accuracy: 88.67\n",
      "Testing loss: 0.38\n",
      "Round 231, Average Train loss 0.354, Average Train Accuracy 89.391\n",
      "Testing accuracy: 89.02\n",
      "Testing loss: 0.37\n",
      "Round 232, Average Train loss 0.338, Average Train Accuracy 90.072\n",
      "Testing accuracy: 88.39\n",
      "Testing loss: 0.38\n",
      "Round 233, Average Train loss 0.341, Average Train Accuracy 89.943\n",
      "Testing accuracy: 88.69\n",
      "Testing loss: 0.38\n",
      "Round 234, Average Train loss 0.333, Average Train Accuracy 90.408\n",
      "Testing accuracy: 88.97\n",
      "Testing loss: 0.37\n",
      "Round 235, Average Train loss 0.357, Average Train Accuracy 88.890\n",
      "Testing accuracy: 89.23\n",
      "Testing loss: 0.37\n",
      "Round 236, Average Train loss 0.363, Average Train Accuracy 89.603\n",
      "Testing accuracy: 88.87\n",
      "Testing loss: 0.37\n",
      "Round 237, Average Train loss 0.325, Average Train Accuracy 91.027\n",
      "Testing accuracy: 88.87\n",
      "Testing loss: 0.37\n",
      "Round 238, Average Train loss 0.326, Average Train Accuracy 90.279\n",
      "Testing accuracy: 88.95\n",
      "Testing loss: 0.37\n",
      "Round 239, Average Train loss 0.325, Average Train Accuracy 90.653\n",
      "Testing accuracy: 88.75\n",
      "Testing loss: 0.37\n",
      "Round 240, Average Train loss 0.329, Average Train Accuracy 90.280\n",
      "Testing accuracy: 88.98\n",
      "Testing loss: 0.37\n",
      "Round 241, Average Train loss 0.344, Average Train Accuracy 89.751\n",
      "Testing accuracy: 88.90\n",
      "Testing loss: 0.37\n",
      "Round 242, Average Train loss 0.336, Average Train Accuracy 90.168\n",
      "Testing accuracy: 89.16\n",
      "Testing loss: 0.37\n",
      "Round 243, Average Train loss 0.344, Average Train Accuracy 90.117\n",
      "Testing accuracy: 88.90\n",
      "Testing loss: 0.37\n",
      "Round 244, Average Train loss 0.339, Average Train Accuracy 89.914\n",
      "Testing accuracy: 89.18\n",
      "Testing loss: 0.36\n",
      "Round 245, Average Train loss 0.339, Average Train Accuracy 90.389\n",
      "Testing accuracy: 89.48\n",
      "Testing loss: 0.36\n",
      "Round 246, Average Train loss 0.344, Average Train Accuracy 90.301\n",
      "Testing accuracy: 88.92\n",
      "Testing loss: 0.37\n",
      "Round 247, Average Train loss 0.336, Average Train Accuracy 90.810\n",
      "Testing accuracy: 89.46\n",
      "Testing loss: 0.36\n",
      "Round 248, Average Train loss 0.352, Average Train Accuracy 89.733\n",
      "Testing accuracy: 89.44\n",
      "Testing loss: 0.36\n",
      "Round 249, Average Train loss 0.361, Average Train Accuracy 89.416\n",
      "Testing accuracy: 89.18\n",
      "Testing loss: 0.36\n",
      "Round 250, Average Train loss 0.342, Average Train Accuracy 90.074\n",
      "Testing accuracy: 89.32\n",
      "Testing loss: 0.36\n",
      "Round 251, Average Train loss 0.343, Average Train Accuracy 89.934\n",
      "Testing accuracy: 88.95\n",
      "Testing loss: 0.37\n",
      "Round 252, Average Train loss 0.316, Average Train Accuracy 90.845\n",
      "Testing accuracy: 88.97\n",
      "Testing loss: 0.37\n",
      "Round 253, Average Train loss 0.331, Average Train Accuracy 89.670\n",
      "Testing accuracy: 89.29\n",
      "Testing loss: 0.36\n",
      "Round 254, Average Train loss 0.319, Average Train Accuracy 91.086\n",
      "Testing accuracy: 89.35\n",
      "Testing loss: 0.36\n",
      "Round 255, Average Train loss 0.328, Average Train Accuracy 90.127\n",
      "Testing accuracy: 89.54\n",
      "Testing loss: 0.35\n",
      "Round 256, Average Train loss 0.341, Average Train Accuracy 90.403\n",
      "Testing accuracy: 89.36\n",
      "Testing loss: 0.36\n",
      "Round 257, Average Train loss 0.333, Average Train Accuracy 90.014\n",
      "Testing accuracy: 89.54\n",
      "Testing loss: 0.35\n",
      "Round 258, Average Train loss 0.332, Average Train Accuracy 90.115\n",
      "Testing accuracy: 89.52\n",
      "Testing loss: 0.35\n",
      "Round 259, Average Train loss 0.323, Average Train Accuracy 90.898\n",
      "Testing accuracy: 89.02\n",
      "Testing loss: 0.36\n",
      "Round 260, Average Train loss 0.350, Average Train Accuracy 89.145\n",
      "Testing accuracy: 89.09\n",
      "Testing loss: 0.36\n",
      "Round 261, Average Train loss 0.337, Average Train Accuracy 89.409\n",
      "Testing accuracy: 89.42\n",
      "Testing loss: 0.36\n",
      "Round 262, Average Train loss 0.353, Average Train Accuracy 89.434\n",
      "Testing accuracy: 89.25\n",
      "Testing loss: 0.36\n",
      "Round 263, Average Train loss 0.329, Average Train Accuracy 90.580\n",
      "Testing accuracy: 89.63\n",
      "Testing loss: 0.36\n",
      "Round 264, Average Train loss 0.307, Average Train Accuracy 91.768\n",
      "Testing accuracy: 89.81\n",
      "Testing loss: 0.35\n",
      "Round 265, Average Train loss 0.356, Average Train Accuracy 90.016\n",
      "Testing accuracy: 89.51\n",
      "Testing loss: 0.35\n",
      "Round 266, Average Train loss 0.335, Average Train Accuracy 90.023\n",
      "Testing accuracy: 89.56\n",
      "Testing loss: 0.35\n",
      "Round 267, Average Train loss 0.331, Average Train Accuracy 90.260\n",
      "Testing accuracy: 89.24\n",
      "Testing loss: 0.36\n",
      "Round 268, Average Train loss 0.355, Average Train Accuracy 88.787\n",
      "Testing accuracy: 89.62\n",
      "Testing loss: 0.35\n",
      "Round 269, Average Train loss 0.328, Average Train Accuracy 90.649\n",
      "Testing accuracy: 89.24\n",
      "Testing loss: 0.36\n",
      "Round 270, Average Train loss 0.329, Average Train Accuracy 89.902\n",
      "Testing accuracy: 89.28\n",
      "Testing loss: 0.36\n",
      "Round 271, Average Train loss 0.322, Average Train Accuracy 90.701\n",
      "Testing accuracy: 89.47\n",
      "Testing loss: 0.35\n",
      "Round 272, Average Train loss 0.320, Average Train Accuracy 90.836\n",
      "Testing accuracy: 89.66\n",
      "Testing loss: 0.35\n",
      "Round 273, Average Train loss 0.323, Average Train Accuracy 90.219\n",
      "Testing accuracy: 89.20\n",
      "Testing loss: 0.36\n",
      "Round 274, Average Train loss 0.321, Average Train Accuracy 90.560\n",
      "Testing accuracy: 89.52\n",
      "Testing loss: 0.35\n",
      "Round 275, Average Train loss 0.331, Average Train Accuracy 90.655\n",
      "Testing accuracy: 89.38\n",
      "Testing loss: 0.36\n",
      "Round 276, Average Train loss 0.339, Average Train Accuracy 89.834\n",
      "Testing accuracy: 89.43\n",
      "Testing loss: 0.35\n",
      "Round 277, Average Train loss 0.325, Average Train Accuracy 90.335\n",
      "Testing accuracy: 89.46\n",
      "Testing loss: 0.35\n",
      "Round 278, Average Train loss 0.321, Average Train Accuracy 90.948\n",
      "Testing accuracy: 88.90\n",
      "Testing loss: 0.36\n",
      "Round 279, Average Train loss 0.315, Average Train Accuracy 90.840\n",
      "Testing accuracy: 89.14\n",
      "Testing loss: 0.35\n",
      "Round 280, Average Train loss 0.309, Average Train Accuracy 90.970\n",
      "Testing accuracy: 88.84\n",
      "Testing loss: 0.36\n",
      "Round 281, Average Train loss 0.318, Average Train Accuracy 90.597\n",
      "Testing accuracy: 89.15\n",
      "Testing loss: 0.36\n",
      "Round 282, Average Train loss 0.293, Average Train Accuracy 91.587\n",
      "Testing accuracy: 89.27\n",
      "Testing loss: 0.35\n",
      "Round 283, Average Train loss 0.329, Average Train Accuracy 90.143\n",
      "Testing accuracy: 89.60\n",
      "Testing loss: 0.35\n",
      "Round 284, Average Train loss 0.314, Average Train Accuracy 91.149\n",
      "Testing accuracy: 89.68\n",
      "Testing loss: 0.35\n",
      "Round 285, Average Train loss 0.305, Average Train Accuracy 91.600\n",
      "Testing accuracy: 89.39\n",
      "Testing loss: 0.35\n",
      "Round 286, Average Train loss 0.321, Average Train Accuracy 90.578\n",
      "Testing accuracy: 89.47\n",
      "Testing loss: 0.35\n",
      "Round 287, Average Train loss 0.339, Average Train Accuracy 90.327\n",
      "Testing accuracy: 89.39\n",
      "Testing loss: 0.35\n",
      "Round 288, Average Train loss 0.324, Average Train Accuracy 90.663\n",
      "Testing accuracy: 89.60\n",
      "Testing loss: 0.35\n",
      "Round 289, Average Train loss 0.259, Average Train Accuracy 93.610\n",
      "Testing accuracy: 89.63\n",
      "Testing loss: 0.35\n",
      "Round 290, Average Train loss 0.315, Average Train Accuracy 90.674\n",
      "Testing accuracy: 89.90\n",
      "Testing loss: 0.34\n",
      "Round 291, Average Train loss 0.318, Average Train Accuracy 90.593\n",
      "Testing accuracy: 89.75\n",
      "Testing loss: 0.35\n",
      "Round 292, Average Train loss 0.315, Average Train Accuracy 90.494\n",
      "Testing accuracy: 89.68\n",
      "Testing loss: 0.35\n",
      "Round 293, Average Train loss 0.303, Average Train Accuracy 91.405\n",
      "Testing accuracy: 89.67\n",
      "Testing loss: 0.35\n",
      "Round 294, Average Train loss 0.300, Average Train Accuracy 91.554\n",
      "Testing accuracy: 89.41\n",
      "Testing loss: 0.35\n",
      "Round 295, Average Train loss 0.303, Average Train Accuracy 91.159\n",
      "Testing accuracy: 89.32\n",
      "Testing loss: 0.35\n",
      "Round 296, Average Train loss 0.330, Average Train Accuracy 90.095\n",
      "Testing accuracy: 88.49\n",
      "Testing loss: 0.37\n",
      "Round 297, Average Train loss 0.332, Average Train Accuracy 90.250\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing accuracy: 88.73\n",
      "Testing loss: 0.37\n",
      "Round 298, Average Train loss 0.326, Average Train Accuracy 90.889\n",
      "Testing accuracy: 88.97\n",
      "Testing loss: 0.36\n",
      "Round 299, Average Train loss 0.295, Average Train Accuracy 91.981\n",
      "Testing accuracy: 89.61\n",
      "Testing loss: 0.35\n",
      "Round 300, Average Train loss 0.320, Average Train Accuracy 90.733\n",
      "Testing accuracy: 89.10\n",
      "Testing loss: 0.36\n",
      "Round 301, Average Train loss 0.307, Average Train Accuracy 90.569\n",
      "Testing accuracy: 89.15\n",
      "Testing loss: 0.36\n",
      "Round 302, Average Train loss 0.335, Average Train Accuracy 89.907\n",
      "Testing accuracy: 89.31\n",
      "Testing loss: 0.35\n",
      "Round 303, Average Train loss 0.316, Average Train Accuracy 91.080\n",
      "Testing accuracy: 89.15\n",
      "Testing loss: 0.35\n",
      "Round 304, Average Train loss 0.309, Average Train Accuracy 91.246\n",
      "Testing accuracy: 89.07\n",
      "Testing loss: 0.35\n",
      "Round 305, Average Train loss 0.318, Average Train Accuracy 89.990\n",
      "Testing accuracy: 89.50\n",
      "Testing loss: 0.34\n",
      "Round 306, Average Train loss 0.325, Average Train Accuracy 90.537\n",
      "Testing accuracy: 89.31\n",
      "Testing loss: 0.35\n",
      "Round 307, Average Train loss 0.318, Average Train Accuracy 90.725\n",
      "Testing accuracy: 89.54\n",
      "Testing loss: 0.35\n",
      "Round 308, Average Train loss 0.303, Average Train Accuracy 91.409\n",
      "Testing accuracy: 89.66\n",
      "Testing loss: 0.34\n",
      "Round 309, Average Train loss 0.272, Average Train Accuracy 92.272\n",
      "Testing accuracy: 89.66\n",
      "Testing loss: 0.35\n",
      "Round 310, Average Train loss 0.313, Average Train Accuracy 91.528\n",
      "Testing accuracy: 89.93\n",
      "Testing loss: 0.34\n",
      "Round 311, Average Train loss 0.327, Average Train Accuracy 90.394\n",
      "Testing accuracy: 90.02\n",
      "Testing loss: 0.34\n",
      "Round 312, Average Train loss 0.305, Average Train Accuracy 91.353\n",
      "Testing accuracy: 89.70\n",
      "Testing loss: 0.34\n",
      "Round 313, Average Train loss 0.303, Average Train Accuracy 91.230\n",
      "Testing accuracy: 89.29\n",
      "Testing loss: 0.35\n",
      "Round 314, Average Train loss 0.292, Average Train Accuracy 91.145\n",
      "Testing accuracy: 88.97\n",
      "Testing loss: 0.35\n",
      "Round 315, Average Train loss 0.306, Average Train Accuracy 90.603\n",
      "Testing accuracy: 88.89\n",
      "Testing loss: 0.36\n",
      "Round 316, Average Train loss 0.294, Average Train Accuracy 90.960\n",
      "Testing accuracy: 88.99\n",
      "Testing loss: 0.36\n",
      "Round 317, Average Train loss 0.305, Average Train Accuracy 90.541\n",
      "Testing accuracy: 88.95\n",
      "Testing loss: 0.36\n",
      "Round 318, Average Train loss 0.321, Average Train Accuracy 90.253\n",
      "Testing accuracy: 88.55\n",
      "Testing loss: 0.36\n",
      "Round 319, Average Train loss 0.319, Average Train Accuracy 90.016\n",
      "Testing accuracy: 89.11\n",
      "Testing loss: 0.35\n",
      "Round 320, Average Train loss 0.303, Average Train Accuracy 90.873\n",
      "Testing accuracy: 89.42\n",
      "Testing loss: 0.34\n",
      "Round 321, Average Train loss 0.299, Average Train Accuracy 90.999\n",
      "Testing accuracy: 89.97\n",
      "Testing loss: 0.34\n",
      "Round 322, Average Train loss 0.318, Average Train Accuracy 90.754\n",
      "Testing accuracy: 89.54\n",
      "Testing loss: 0.34\n",
      "Round 323, Average Train loss 0.313, Average Train Accuracy 90.103\n",
      "Testing accuracy: 89.36\n",
      "Testing loss: 0.35\n",
      "Round 324, Average Train loss 0.310, Average Train Accuracy 90.998\n",
      "Testing accuracy: 89.53\n",
      "Testing loss: 0.34\n",
      "Round 325, Average Train loss 0.280, Average Train Accuracy 91.591\n",
      "Testing accuracy: 89.94\n",
      "Testing loss: 0.33\n",
      "Round 326, Average Train loss 0.311, Average Train Accuracy 90.878\n",
      "Testing accuracy: 89.97\n",
      "Testing loss: 0.34\n",
      "Round 327, Average Train loss 0.312, Average Train Accuracy 91.005\n",
      "Testing accuracy: 89.97\n",
      "Testing loss: 0.33\n",
      "Round 328, Average Train loss 0.299, Average Train Accuracy 91.568\n",
      "Testing accuracy: 89.36\n",
      "Testing loss: 0.34\n",
      "Round 329, Average Train loss 0.292, Average Train Accuracy 91.656\n",
      "Testing accuracy: 89.53\n",
      "Testing loss: 0.34\n",
      "Round 330, Average Train loss 0.301, Average Train Accuracy 91.089\n",
      "Testing accuracy: 89.54\n",
      "Testing loss: 0.34\n",
      "Round 331, Average Train loss 0.301, Average Train Accuracy 90.897\n",
      "Testing accuracy: 89.75\n",
      "Testing loss: 0.34\n",
      "Round 332, Average Train loss 0.298, Average Train Accuracy 91.475\n",
      "Testing accuracy: 89.60\n",
      "Testing loss: 0.34\n",
      "Round 333, Average Train loss 0.296, Average Train Accuracy 91.230\n",
      "Testing accuracy: 89.64\n",
      "Testing loss: 0.34\n",
      "Round 334, Average Train loss 0.276, Average Train Accuracy 91.903\n",
      "Testing accuracy: 89.90\n",
      "Testing loss: 0.33\n",
      "Round 335, Average Train loss 0.302, Average Train Accuracy 90.845\n",
      "Testing accuracy: 89.81\n",
      "Testing loss: 0.33\n",
      "Round 336, Average Train loss 0.289, Average Train Accuracy 91.689\n",
      "Testing accuracy: 89.59\n",
      "Testing loss: 0.34\n",
      "Round 337, Average Train loss 0.305, Average Train Accuracy 91.007\n",
      "Testing accuracy: 89.83\n",
      "Testing loss: 0.33\n",
      "Round 338, Average Train loss 0.285, Average Train Accuracy 92.090\n",
      "Testing accuracy: 90.14\n",
      "Testing loss: 0.33\n",
      "Round 339, Average Train loss 0.304, Average Train Accuracy 90.956\n",
      "Testing accuracy: 90.03\n",
      "Testing loss: 0.33\n",
      "Round 340, Average Train loss 0.300, Average Train Accuracy 91.163\n",
      "Testing accuracy: 90.10\n",
      "Testing loss: 0.33\n",
      "Round 341, Average Train loss 0.306, Average Train Accuracy 91.091\n",
      "Testing accuracy: 89.93\n",
      "Testing loss: 0.33\n",
      "Round 342, Average Train loss 0.276, Average Train Accuracy 92.390\n",
      "Testing accuracy: 89.75\n",
      "Testing loss: 0.34\n",
      "Round 343, Average Train loss 0.289, Average Train Accuracy 91.690\n",
      "Testing accuracy: 89.59\n",
      "Testing loss: 0.34\n",
      "Round 344, Average Train loss 0.307, Average Train Accuracy 90.628\n",
      "Testing accuracy: 89.88\n",
      "Testing loss: 0.34\n",
      "Round 345, Average Train loss 0.288, Average Train Accuracy 91.426\n",
      "Testing accuracy: 90.23\n",
      "Testing loss: 0.33\n",
      "Round 346, Average Train loss 0.293, Average Train Accuracy 92.259\n",
      "Testing accuracy: 90.14\n",
      "Testing loss: 0.33\n",
      "Round 347, Average Train loss 0.327, Average Train Accuracy 90.117\n",
      "Testing accuracy: 90.33\n",
      "Testing loss: 0.32\n",
      "Round 348, Average Train loss 0.315, Average Train Accuracy 90.975\n",
      "Testing accuracy: 90.29\n",
      "Testing loss: 0.32\n",
      "Round 349, Average Train loss 0.302, Average Train Accuracy 90.689\n",
      "Testing accuracy: 90.27\n",
      "Testing loss: 0.32\n",
      "Round 350, Average Train loss 0.281, Average Train Accuracy 91.632\n",
      "Testing accuracy: 90.17\n",
      "Testing loss: 0.33\n",
      "Round 351, Average Train loss 0.271, Average Train Accuracy 92.221\n",
      "Testing accuracy: 90.33\n",
      "Testing loss: 0.32\n",
      "Round 352, Average Train loss 0.314, Average Train Accuracy 90.915\n",
      "Testing accuracy: 90.03\n",
      "Testing loss: 0.33\n",
      "Round 353, Average Train loss 0.324, Average Train Accuracy 90.331\n",
      "Testing accuracy: 90.11\n",
      "Testing loss: 0.33\n",
      "Round 354, Average Train loss 0.277, Average Train Accuracy 92.035\n",
      "Testing accuracy: 90.10\n",
      "Testing loss: 0.33\n",
      "Round 355, Average Train loss 0.294, Average Train Accuracy 91.258\n",
      "Testing accuracy: 90.13\n",
      "Testing loss: 0.33\n",
      "Round 356, Average Train loss 0.303, Average Train Accuracy 91.032\n",
      "Testing accuracy: 90.18\n",
      "Testing loss: 0.33\n",
      "Round 357, Average Train loss 0.273, Average Train Accuracy 92.389\n",
      "Testing accuracy: 89.65\n",
      "Testing loss: 0.34\n",
      "Round 358, Average Train loss 0.317, Average Train Accuracy 90.933\n",
      "Testing accuracy: 89.90\n",
      "Testing loss: 0.33\n",
      "Round 359, Average Train loss 0.271, Average Train Accuracy 92.076\n",
      "Testing accuracy: 89.91\n",
      "Testing loss: 0.33\n",
      "Round 360, Average Train loss 0.279, Average Train Accuracy 91.944\n",
      "Testing accuracy: 89.87\n",
      "Testing loss: 0.34\n",
      "Round 361, Average Train loss 0.269, Average Train Accuracy 92.825\n",
      "Testing accuracy: 90.12\n",
      "Testing loss: 0.33\n",
      "Round 362, Average Train loss 0.286, Average Train Accuracy 91.355\n",
      "Testing accuracy: 89.95\n",
      "Testing loss: 0.33\n",
      "Round 363, Average Train loss 0.280, Average Train Accuracy 92.293\n",
      "Testing accuracy: 89.97\n",
      "Testing loss: 0.33\n",
      "Round 364, Average Train loss 0.279, Average Train Accuracy 92.221\n",
      "Testing accuracy: 89.80\n",
      "Testing loss: 0.33\n",
      "Round 365, Average Train loss 0.300, Average Train Accuracy 91.222\n",
      "Testing accuracy: 90.36\n",
      "Testing loss: 0.32\n",
      "Round 366, Average Train loss 0.299, Average Train Accuracy 91.142\n",
      "Testing accuracy: 90.32\n",
      "Testing loss: 0.32\n",
      "Round 367, Average Train loss 0.289, Average Train Accuracy 91.380\n",
      "Testing accuracy: 90.33\n",
      "Testing loss: 0.32\n",
      "Round 368, Average Train loss 0.268, Average Train Accuracy 92.531\n",
      "Testing accuracy: 90.31\n",
      "Testing loss: 0.32\n",
      "Round 369, Average Train loss 0.268, Average Train Accuracy 92.472\n",
      "Testing accuracy: 90.11\n",
      "Testing loss: 0.33\n",
      "Round 370, Average Train loss 0.276, Average Train Accuracy 91.858\n",
      "Testing accuracy: 90.31\n",
      "Testing loss: 0.32\n",
      "Round 371, Average Train loss 0.291, Average Train Accuracy 91.946\n",
      "Testing accuracy: 90.33\n",
      "Testing loss: 0.32\n",
      "Round 372, Average Train loss 0.288, Average Train Accuracy 91.623\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing accuracy: 90.73\n",
      "Testing loss: 0.32\n",
      "Round 373, Average Train loss 0.298, Average Train Accuracy 91.076\n",
      "Testing accuracy: 90.61\n",
      "Testing loss: 0.32\n",
      "Round 374, Average Train loss 0.309, Average Train Accuracy 90.832\n",
      "Testing accuracy: 90.61\n",
      "Testing loss: 0.32\n",
      "Round 375, Average Train loss 0.281, Average Train Accuracy 91.725\n",
      "Testing accuracy: 90.32\n",
      "Testing loss: 0.32\n",
      "Round 376, Average Train loss 0.276, Average Train Accuracy 91.955\n",
      "Testing accuracy: 90.08\n",
      "Testing loss: 0.33\n",
      "Round 377, Average Train loss 0.300, Average Train Accuracy 91.142\n",
      "Testing accuracy: 90.27\n",
      "Testing loss: 0.32\n",
      "Round 378, Average Train loss 0.285, Average Train Accuracy 91.367\n",
      "Testing accuracy: 90.32\n",
      "Testing loss: 0.32\n",
      "Round 379, Average Train loss 0.299, Average Train Accuracy 91.442\n",
      "Testing accuracy: 90.59\n",
      "Testing loss: 0.32\n",
      "Round 380, Average Train loss 0.283, Average Train Accuracy 92.395\n",
      "Testing accuracy: 90.56\n",
      "Testing loss: 0.32\n",
      "Round 381, Average Train loss 0.275, Average Train Accuracy 92.975\n",
      "Testing accuracy: 90.06\n",
      "Testing loss: 0.33\n",
      "Round 382, Average Train loss 0.288, Average Train Accuracy 91.504\n",
      "Testing accuracy: 90.24\n",
      "Testing loss: 0.32\n",
      "Round 383, Average Train loss 0.298, Average Train Accuracy 91.343\n",
      "Testing accuracy: 90.52\n",
      "Testing loss: 0.32\n",
      "Round 384, Average Train loss 0.276, Average Train Accuracy 91.721\n",
      "Testing accuracy: 90.57\n",
      "Testing loss: 0.32\n",
      "Round 385, Average Train loss 0.258, Average Train Accuracy 92.438\n",
      "Testing accuracy: 90.40\n",
      "Testing loss: 0.32\n",
      "Round 386, Average Train loss 0.299, Average Train Accuracy 91.777\n",
      "Testing accuracy: 90.82\n",
      "Testing loss: 0.31\n",
      "Round 387, Average Train loss 0.299, Average Train Accuracy 91.094\n",
      "Testing accuracy: 90.56\n",
      "Testing loss: 0.32\n",
      "Round 388, Average Train loss 0.283, Average Train Accuracy 91.858\n",
      "Testing accuracy: 90.39\n",
      "Testing loss: 0.32\n",
      "Round 389, Average Train loss 0.290, Average Train Accuracy 91.915\n",
      "Testing accuracy: 90.11\n",
      "Testing loss: 0.32\n",
      "Round 390, Average Train loss 0.271, Average Train Accuracy 92.352\n",
      "Testing accuracy: 90.04\n",
      "Testing loss: 0.33\n",
      "Round 391, Average Train loss 0.275, Average Train Accuracy 91.840\n",
      "Testing accuracy: 90.22\n",
      "Testing loss: 0.32\n",
      "Round 392, Average Train loss 0.271, Average Train Accuracy 92.413\n",
      "Testing accuracy: 90.55\n",
      "Testing loss: 0.32\n",
      "Round 393, Average Train loss 0.265, Average Train Accuracy 92.403\n",
      "Testing accuracy: 90.49\n",
      "Testing loss: 0.32\n",
      "Round 394, Average Train loss 0.289, Average Train Accuracy 91.381\n",
      "Testing accuracy: 90.56\n",
      "Testing loss: 0.31\n",
      "Round 395, Average Train loss 0.315, Average Train Accuracy 91.039\n",
      "Testing accuracy: 90.45\n",
      "Testing loss: 0.32\n",
      "Round 396, Average Train loss 0.271, Average Train Accuracy 92.335\n",
      "Testing accuracy: 90.51\n",
      "Testing loss: 0.32\n",
      "Round 397, Average Train loss 0.269, Average Train Accuracy 92.586\n",
      "Testing accuracy: 90.64\n",
      "Testing loss: 0.31\n",
      "Round 398, Average Train loss 0.306, Average Train Accuracy 89.906\n",
      "Testing accuracy: 90.56\n",
      "Testing loss: 0.32\n",
      "Round 399, Average Train loss 0.268, Average Train Accuracy 92.576\n",
      "Testing accuracy: 90.28\n",
      "Testing loss: 0.32\n",
      "Round 400, Average Train loss 0.278, Average Train Accuracy 91.724\n",
      "Testing accuracy: 90.51\n",
      "Testing loss: 0.31\n",
      "Round 401, Average Train loss 0.282, Average Train Accuracy 91.649\n",
      "Testing accuracy: 90.91\n",
      "Testing loss: 0.31\n",
      "Round 402, Average Train loss 0.262, Average Train Accuracy 92.616\n",
      "Testing accuracy: 90.49\n",
      "Testing loss: 0.31\n",
      "Round 403, Average Train loss 0.291, Average Train Accuracy 90.908\n",
      "Testing accuracy: 90.10\n",
      "Testing loss: 0.32\n",
      "Round 404, Average Train loss 0.292, Average Train Accuracy 91.044\n",
      "Testing accuracy: 90.46\n",
      "Testing loss: 0.31\n",
      "Round 405, Average Train loss 0.300, Average Train Accuracy 90.594\n",
      "Testing accuracy: 90.29\n",
      "Testing loss: 0.32\n",
      "Round 406, Average Train loss 0.264, Average Train Accuracy 92.310\n",
      "Testing accuracy: 90.08\n",
      "Testing loss: 0.32\n",
      "Round 407, Average Train loss 0.289, Average Train Accuracy 91.329\n",
      "Testing accuracy: 89.91\n",
      "Testing loss: 0.33\n",
      "Round 408, Average Train loss 0.273, Average Train Accuracy 92.294\n",
      "Testing accuracy: 89.59\n",
      "Testing loss: 0.33\n",
      "Round 409, Average Train loss 0.260, Average Train Accuracy 92.279\n",
      "Testing accuracy: 89.79\n",
      "Testing loss: 0.33\n",
      "Round 410, Average Train loss 0.274, Average Train Accuracy 91.795\n",
      "Testing accuracy: 90.16\n",
      "Testing loss: 0.32\n",
      "Round 411, Average Train loss 0.276, Average Train Accuracy 91.732\n",
      "Testing accuracy: 90.28\n",
      "Testing loss: 0.32\n",
      "Round 412, Average Train loss 0.278, Average Train Accuracy 92.002\n",
      "Testing accuracy: 90.40\n",
      "Testing loss: 0.32\n",
      "Round 413, Average Train loss 0.254, Average Train Accuracy 92.516\n",
      "Testing accuracy: 90.23\n",
      "Testing loss: 0.32\n",
      "Round 414, Average Train loss 0.277, Average Train Accuracy 91.635\n",
      "Testing accuracy: 90.50\n",
      "Testing loss: 0.32\n",
      "Round 415, Average Train loss 0.271, Average Train Accuracy 92.190\n",
      "Testing accuracy: 90.49\n",
      "Testing loss: 0.32\n",
      "Round 416, Average Train loss 0.258, Average Train Accuracy 92.595\n",
      "Testing accuracy: 90.43\n",
      "Testing loss: 0.32\n",
      "Round 417, Average Train loss 0.265, Average Train Accuracy 92.800\n",
      "Testing accuracy: 90.25\n",
      "Testing loss: 0.32\n",
      "Round 418, Average Train loss 0.273, Average Train Accuracy 91.887\n",
      "Testing accuracy: 90.21\n",
      "Testing loss: 0.32\n",
      "Round 419, Average Train loss 0.271, Average Train Accuracy 91.879\n",
      "Testing accuracy: 90.18\n",
      "Testing loss: 0.32\n",
      "Round 420, Average Train loss 0.289, Average Train Accuracy 91.754\n",
      "Testing accuracy: 90.41\n",
      "Testing loss: 0.32\n",
      "Round 421, Average Train loss 0.265, Average Train Accuracy 92.311\n",
      "Testing accuracy: 90.09\n",
      "Testing loss: 0.32\n",
      "Round 422, Average Train loss 0.290, Average Train Accuracy 91.329\n",
      "Testing accuracy: 89.88\n",
      "Testing loss: 0.33\n",
      "Round 423, Average Train loss 0.279, Average Train Accuracy 92.146\n",
      "Testing accuracy: 89.98\n",
      "Testing loss: 0.32\n",
      "Round 424, Average Train loss 0.279, Average Train Accuracy 91.261\n",
      "Testing accuracy: 89.84\n",
      "Testing loss: 0.33\n",
      "Round 425, Average Train loss 0.247, Average Train Accuracy 92.939\n",
      "Testing accuracy: 90.10\n",
      "Testing loss: 0.32\n",
      "Round 426, Average Train loss 0.279, Average Train Accuracy 92.481\n",
      "Testing accuracy: 90.12\n",
      "Testing loss: 0.32\n",
      "Round 427, Average Train loss 0.292, Average Train Accuracy 91.446\n",
      "Testing accuracy: 90.33\n",
      "Testing loss: 0.32\n",
      "Round 428, Average Train loss 0.263, Average Train Accuracy 92.130\n",
      "Testing accuracy: 90.05\n",
      "Testing loss: 0.32\n",
      "Round 429, Average Train loss 0.306, Average Train Accuracy 90.941\n",
      "Testing accuracy: 89.88\n",
      "Testing loss: 0.33\n",
      "Round 430, Average Train loss 0.288, Average Train Accuracy 90.815\n",
      "Testing accuracy: 90.38\n",
      "Testing loss: 0.32\n",
      "Round 431, Average Train loss 0.264, Average Train Accuracy 92.088\n",
      "Testing accuracy: 90.57\n",
      "Testing loss: 0.31\n",
      "Round 432, Average Train loss 0.252, Average Train Accuracy 92.451\n",
      "Testing accuracy: 90.55\n",
      "Testing loss: 0.31\n",
      "Round 433, Average Train loss 0.249, Average Train Accuracy 93.318\n",
      "Testing accuracy: 90.55\n",
      "Testing loss: 0.31\n",
      "Round 434, Average Train loss 0.263, Average Train Accuracy 92.489\n",
      "Testing accuracy: 90.61\n",
      "Testing loss: 0.31\n",
      "Round 435, Average Train loss 0.271, Average Train Accuracy 92.119\n",
      "Testing accuracy: 90.43\n",
      "Testing loss: 0.31\n",
      "Round 436, Average Train loss 0.278, Average Train Accuracy 91.860\n",
      "Testing accuracy: 90.60\n",
      "Testing loss: 0.31\n",
      "Round 437, Average Train loss 0.239, Average Train Accuracy 93.266\n",
      "Testing accuracy: 90.78\n",
      "Testing loss: 0.31\n",
      "Round 438, Average Train loss 0.282, Average Train Accuracy 91.825\n",
      "Testing accuracy: 90.81\n",
      "Testing loss: 0.31\n",
      "Round 439, Average Train loss 0.268, Average Train Accuracy 92.612\n",
      "Testing accuracy: 90.96\n",
      "Testing loss: 0.30\n",
      "Round 440, Average Train loss 0.271, Average Train Accuracy 91.993\n",
      "Testing accuracy: 90.59\n",
      "Testing loss: 0.31\n",
      "Round 441, Average Train loss 0.258, Average Train Accuracy 92.291\n",
      "Testing accuracy: 90.62\n",
      "Testing loss: 0.31\n",
      "Round 442, Average Train loss 0.252, Average Train Accuracy 92.775\n",
      "Testing accuracy: 90.60\n",
      "Testing loss: 0.31\n",
      "Round 443, Average Train loss 0.259, Average Train Accuracy 92.415\n",
      "Testing accuracy: 90.65\n",
      "Testing loss: 0.31\n",
      "Round 444, Average Train loss 0.235, Average Train Accuracy 93.407\n",
      "Testing accuracy: 90.80\n",
      "Testing loss: 0.31\n",
      "Round 445, Average Train loss 0.274, Average Train Accuracy 91.907\n",
      "Testing accuracy: 90.47\n",
      "Testing loss: 0.31\n",
      "Round 446, Average Train loss 0.241, Average Train Accuracy 92.612\n",
      "Testing accuracy: 90.52\n",
      "Testing loss: 0.31\n",
      "Round 447, Average Train loss 0.273, Average Train Accuracy 92.106\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing accuracy: 90.49\n",
      "Testing loss: 0.31\n",
      "Round 448, Average Train loss 0.263, Average Train Accuracy 92.585\n",
      "Testing accuracy: 90.63\n",
      "Testing loss: 0.31\n",
      "Round 449, Average Train loss 0.243, Average Train Accuracy 92.627\n",
      "Testing accuracy: 90.76\n",
      "Testing loss: 0.31\n",
      "Round 450, Average Train loss 0.275, Average Train Accuracy 91.858\n",
      "Testing accuracy: 90.60\n",
      "Testing loss: 0.31\n",
      "Round 451, Average Train loss 0.270, Average Train Accuracy 91.893\n",
      "Testing accuracy: 90.53\n",
      "Testing loss: 0.31\n",
      "Round 452, Average Train loss 0.263, Average Train Accuracy 91.820\n",
      "Testing accuracy: 90.39\n",
      "Testing loss: 0.31\n",
      "Round 453, Average Train loss 0.258, Average Train Accuracy 91.921\n",
      "Testing accuracy: 90.61\n",
      "Testing loss: 0.31\n",
      "Round 454, Average Train loss 0.282, Average Train Accuracy 92.006\n",
      "Testing accuracy: 90.91\n",
      "Testing loss: 0.30\n",
      "Round 455, Average Train loss 0.254, Average Train Accuracy 92.754\n",
      "Testing accuracy: 91.10\n",
      "Testing loss: 0.30\n",
      "Round 456, Average Train loss 0.261, Average Train Accuracy 92.584\n",
      "Testing accuracy: 91.08\n",
      "Testing loss: 0.30\n",
      "Round 457, Average Train loss 0.260, Average Train Accuracy 92.155\n",
      "Testing accuracy: 91.06\n",
      "Testing loss: 0.30\n",
      "Round 458, Average Train loss 0.245, Average Train Accuracy 93.293\n",
      "Testing accuracy: 90.96\n",
      "Testing loss: 0.30\n",
      "Round 459, Average Train loss 0.273, Average Train Accuracy 91.928\n",
      "Testing accuracy: 90.88\n",
      "Testing loss: 0.30\n",
      "Round 460, Average Train loss 0.273, Average Train Accuracy 91.963\n",
      "Testing accuracy: 91.05\n",
      "Testing loss: 0.30\n",
      "Round 461, Average Train loss 0.289, Average Train Accuracy 91.433\n",
      "Testing accuracy: 90.92\n",
      "Testing loss: 0.30\n",
      "Round 462, Average Train loss 0.237, Average Train Accuracy 93.317\n",
      "Testing accuracy: 91.02\n",
      "Testing loss: 0.30\n",
      "Round 463, Average Train loss 0.286, Average Train Accuracy 92.386\n",
      "Testing accuracy: 90.78\n",
      "Testing loss: 0.30\n",
      "Round 464, Average Train loss 0.277, Average Train Accuracy 91.893\n",
      "Testing accuracy: 90.74\n",
      "Testing loss: 0.31\n",
      "Round 465, Average Train loss 0.262, Average Train Accuracy 92.549\n",
      "Testing accuracy: 90.94\n",
      "Testing loss: 0.30\n",
      "Round 466, Average Train loss 0.248, Average Train Accuracy 92.777\n",
      "Testing accuracy: 90.79\n",
      "Testing loss: 0.30\n",
      "Round 467, Average Train loss 0.255, Average Train Accuracy 92.438\n",
      "Testing accuracy: 90.75\n",
      "Testing loss: 0.30\n",
      "Round 468, Average Train loss 0.236, Average Train Accuracy 93.075\n",
      "Testing accuracy: 90.62\n",
      "Testing loss: 0.31\n",
      "Round 469, Average Train loss 0.235, Average Train Accuracy 93.447\n",
      "Testing accuracy: 90.45\n",
      "Testing loss: 0.31\n",
      "Round 470, Average Train loss 0.271, Average Train Accuracy 92.422\n",
      "Testing accuracy: 90.80\n",
      "Testing loss: 0.30\n",
      "Round 471, Average Train loss 0.254, Average Train Accuracy 92.835\n",
      "Testing accuracy: 90.99\n",
      "Testing loss: 0.30\n",
      "Round 472, Average Train loss 0.272, Average Train Accuracy 92.367\n",
      "Testing accuracy: 91.13\n",
      "Testing loss: 0.30\n",
      "Round 473, Average Train loss 0.262, Average Train Accuracy 92.471\n",
      "Testing accuracy: 90.99\n",
      "Testing loss: 0.30\n",
      "Round 474, Average Train loss 0.283, Average Train Accuracy 91.669\n",
      "Testing accuracy: 90.80\n",
      "Testing loss: 0.30\n",
      "Round 475, Average Train loss 0.247, Average Train Accuracy 92.912\n",
      "Testing accuracy: 90.51\n",
      "Testing loss: 0.31\n",
      "Round 476, Average Train loss 0.250, Average Train Accuracy 93.358\n",
      "Testing accuracy: 90.60\n",
      "Testing loss: 0.31\n",
      "Round 477, Average Train loss 0.268, Average Train Accuracy 92.170\n",
      "Testing accuracy: 90.66\n",
      "Testing loss: 0.31\n",
      "Round 478, Average Train loss 0.272, Average Train Accuracy 91.847\n",
      "Testing accuracy: 90.59\n",
      "Testing loss: 0.31\n",
      "Round 479, Average Train loss 0.259, Average Train Accuracy 92.390\n",
      "Testing accuracy: 90.48\n",
      "Testing loss: 0.31\n",
      "Round 480, Average Train loss 0.278, Average Train Accuracy 91.894\n",
      "Testing accuracy: 90.28\n",
      "Testing loss: 0.31\n",
      "Round 481, Average Train loss 0.251, Average Train Accuracy 92.487\n",
      "Testing accuracy: 90.42\n",
      "Testing loss: 0.31\n",
      "Round 482, Average Train loss 0.242, Average Train Accuracy 92.721\n",
      "Testing accuracy: 90.43\n",
      "Testing loss: 0.31\n",
      "Round 483, Average Train loss 0.255, Average Train Accuracy 92.074\n",
      "Testing accuracy: 90.54\n",
      "Testing loss: 0.31\n",
      "Round 484, Average Train loss 0.249, Average Train Accuracy 92.667\n",
      "Testing accuracy: 90.48\n",
      "Testing loss: 0.31\n",
      "Round 485, Average Train loss 0.255, Average Train Accuracy 92.362\n",
      "Testing accuracy: 90.49\n",
      "Testing loss: 0.31\n",
      "Round 486, Average Train loss 0.279, Average Train Accuracy 91.803\n",
      "Testing accuracy: 90.89\n",
      "Testing loss: 0.30\n",
      "Round 487, Average Train loss 0.260, Average Train Accuracy 92.591\n",
      "Testing accuracy: 90.95\n",
      "Testing loss: 0.30\n",
      "Round 488, Average Train loss 0.222, Average Train Accuracy 93.555\n",
      "Testing accuracy: 90.78\n",
      "Testing loss: 0.30\n",
      "Round 489, Average Train loss 0.272, Average Train Accuracy 92.504\n",
      "Testing accuracy: 90.95\n",
      "Testing loss: 0.30\n",
      "Round 490, Average Train loss 0.230, Average Train Accuracy 93.590\n",
      "Testing accuracy: 91.06\n",
      "Testing loss: 0.30\n",
      "Round 491, Average Train loss 0.257, Average Train Accuracy 92.452\n",
      "Testing accuracy: 90.89\n",
      "Testing loss: 0.30\n",
      "Round 492, Average Train loss 0.257, Average Train Accuracy 92.568\n",
      "Testing accuracy: 91.17\n",
      "Testing loss: 0.30\n",
      "Round 493, Average Train loss 0.257, Average Train Accuracy 92.315\n",
      "Testing accuracy: 91.19\n",
      "Testing loss: 0.30\n",
      "Round 494, Average Train loss 0.239, Average Train Accuracy 93.059\n",
      "Testing accuracy: 90.86\n",
      "Testing loss: 0.30\n",
      "Round 495, Average Train loss 0.243, Average Train Accuracy 92.861\n",
      "Testing accuracy: 90.55\n",
      "Testing loss: 0.31\n",
      "Round 496, Average Train loss 0.239, Average Train Accuracy 93.001\n",
      "Testing accuracy: 90.88\n",
      "Testing loss: 0.30\n",
      "Round 497, Average Train loss 0.252, Average Train Accuracy 93.074\n",
      "Testing accuracy: 90.94\n",
      "Testing loss: 0.30\n",
      "Round 498, Average Train loss 0.262, Average Train Accuracy 92.573\n",
      "Testing accuracy: 90.75\n",
      "Testing loss: 0.30\n",
      "Round 499, Average Train loss 0.242, Average Train Accuracy 92.750\n",
      "Testing accuracy: 90.96\n",
      "Testing loss: 0.30\n"
     ]
    }
   ],
   "source": [
    "Data_num_S_N_T = np.zeros((N, T, wS.shape[0]))\n",
    "for u in range(N):\n",
    "    for t in range(T):\n",
    "        Data_num_S_N_T[u, t] = wS\n",
    "        for i in range(m):\n",
    "            \n",
    "                    j = j_N_T[u, t, i]#np.random.choice(n + 1, p=x_t_N_T[u, t, :, i])\n",
    "                    \n",
    "                    if j != 0:\n",
    "                        j -= 1\n",
    "                        Data_num_S_N_T[u, t, j] += y_N_T[u, t, i] #upload to device j-1\n",
    "                        Data_num_S_N_T[u, t, j] = min(Data_num_S_N_T[u, t, j], BS_N[u, j])\n",
    "\n",
    "\n",
    "from fed_train import fed_T\n",
    "import time\n",
    "save_time = str(time.time())\n",
    "import torch\n",
    "torch.manual_seed(seed)\n",
    "np.random.seed(seed)\n",
    "\n",
    "total_loss_train = []\n",
    "total_accuracy_train = []\n",
    "\n",
    "total_acc_test = []\n",
    "total_loss_test = []\n",
    "\n",
    "for u in range(N):\n",
    "    L_T = []\n",
    "    for t in range(T):\n",
    "        L = list(Data_num_S_N_T[u, t])\n",
    "        L_T.append(L)\n",
    "    L_T = np.array(L_T).astype(int)\n",
    "\n",
    "\n",
    "\n",
    "    loss_train, accuracy_train, acc_tests, loss_tests, args = fed_T(L_T, \"mnist\", False)\n",
    "    if len(total_loss_train) == 0:\n",
    "        total_loss_train = np.array(loss_train)\n",
    "    else:\n",
    "        total_loss_train += np.array(loss_train)\n",
    "        \n",
    "    if len(total_accuracy_train) == 0:\n",
    "        total_accuracy_train = np.array(accuracy_train)\n",
    "    else:\n",
    "        total_accuracy_train += np.array(accuracy_train)\n",
    "        \n",
    "        \n",
    "    if len(total_acc_test) == 0:\n",
    "        total_acc_test = np.array(acc_tests)\n",
    "    else:\n",
    "        total_acc_test += np.array(acc_tests)\n",
    "    \n",
    "    if len(total_loss_test) == 0:\n",
    "        total_loss_test = np.array(loss_tests)\n",
    "    else:\n",
    "        total_loss_test += np.array(loss_tests)\n",
    "\n",
    "    \n",
    "total_loss_train /= N\n",
    "total_accuracy_train /= N\n",
    "total_acc_test /= N\n",
    "total_loss_test /= N\n",
    "\n",
    "np.save(\"hard_iid\" + str(args.iid) + \"total_loss_train\", total_loss_train)\n",
    "np.save(\"hard_iid\" + str(args.iid) + \"total_accuracy_train\", total_accuracy_train)\n",
    "np.save(\"hard_iid\" + str(args.iid) + \"total_acc_test\", total_acc_test)\n",
    "np.save(\"hard_iid\" + str(args.iid) + \"total_loss_test\", total_loss_test)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "weird-heart",
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.manual_seed(seed)\n",
    "np.random.seed(seed)\n",
    "Data_num_S_N_T = np.zeros((N, T, wS.shape[0]))\n",
    "for u in range(N):\n",
    "    for t in range(T):\n",
    "        Data_num_S_N_T[u, t] = wS\n",
    "        for i in range(m):\n",
    "            \n",
    "                    j = np.random.choice(n + 1)\n",
    "                    \n",
    "                    if j != 0:\n",
    "                        j -= 1\n",
    "                        Data_num_S_N_T[u, t, j] += y_N_T[u, t, i] #upload to device j-1\n",
    "                        Data_num_S_N_T[u, t, j] = min(Data_num_S_N_T[u, t, j], BS_N[u, j])\n",
    "\n",
    "\n",
    "from fed_train import fed_T\n",
    "import time\n",
    "save_time = str(time.time())\n",
    "import torch\n",
    "\n",
    "\n",
    "total_loss_train = []\n",
    "total_accuracy_train = []\n",
    "\n",
    "total_acc_test = []\n",
    "total_loss_test = []\n",
    "\n",
    "for u in range(N):\n",
    "    L_T = []\n",
    "    for t in range(T):\n",
    "        L = list(Data_num_S_N_T[u, t])\n",
    "        L_T.append(L)\n",
    "    L_T = np.array(L_T).astype(int)\n",
    "\n",
    "\n",
    "\n",
    "    loss_train, accuracy_train, acc_tests, loss_tests, args = fed_T(L_T, \"mnist\", False)\n",
    "    if len(total_loss_train) == 0:\n",
    "        total_loss_train = np.array(loss_train)\n",
    "    else:\n",
    "        total_loss_train += np.array(loss_train)\n",
    "        \n",
    "    if len(total_accuracy_train) == 0:\n",
    "        total_accuracy_train = np.array(accuracy_train)\n",
    "    else:\n",
    "        total_accuracy_train += np.array(accuracy_train)\n",
    "        \n",
    "        \n",
    "    if len(total_acc_test) == 0:\n",
    "        total_acc_test = np.array(acc_tests)\n",
    "    else:\n",
    "        total_acc_test += np.array(acc_tests)\n",
    "    \n",
    "    if len(total_loss_test) == 0:\n",
    "        total_loss_test = np.array(loss_tests)\n",
    "    else:\n",
    "        total_loss_test += np.array(loss_tests)\n",
    "\n",
    "    \n",
    "total_loss_train /= N\n",
    "total_accuracy_train /= N\n",
    "total_acc_test /= N\n",
    "total_loss_test /= N\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "np.save(\"random_hard_iid\" + str(args.iid) + \"total_loss_train\", total_loss_train)\n",
    "np.save(\"random_hard_iid\" + str(args.iid) + \"total_accuracy_train\", total_accuracy_train)\n",
    "np.save(\"random_hard_iid\" + str(args.iid) + \"total_acc_test\", total_acc_test)\n",
    "np.save(\"random_hard_iid\" + str(args.iid) + \"total_loss_test\", total_loss_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "impaired-serum",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
